l#g TdZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlmZmZddlmZmZmZddlmZmZddlmZddl m!Z!ddl"m#Z#dd l$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-dd l.m/Z/dd l0m1Z1dd l2m3Z3dd l4m5Z5m6Z6m7Z7ddl8m9Z9m:Z:ddl;mZ>e#e?Z@dZAdZBdZCe jDdZEdZFdZGdZHdZI e jJ ZKe-eLeMeNeOde'eLe%fe)e%fZPGddeQZRGddeQZSdZTde jUd eMd!e&fd"ZVd!ePfd#ZWe6eSeTeH$d%eLd!ePfd&ZXid'd%eLd(eNfd)ZYe6eSeTeH$d%eLd*eNd(eNd!eOfd+ZZed,d-d%eLd(eNfd.Z[d/e&fd0Z\e6eSeTeH$d,dd1d%eLd2e jUd3eMd!eLfd4Z]ed5d%d6gZ^d7e%d!e+e^fd8Z_d9Z`eEfd:e jDd;e jDd!dfd<ZaGd=d>ZbdDd?ZcebjdZeebjfZf dEdAe*eLd!dfdBZgdCZhdS)FaPUtilities for managing local file storage synchronised with a remote server. Files are divided into types: signatures, modsecurity bundles, ip white lists, etc. Each type is represented by an Index instance. Index has a local subdirectory and a description that contains its files' metadata used to decide if the update is necessary. N) defaultdict namedtuple) ExitStacksuppresscontextmanager) formatdateparsedate_to_datetime)GzipFile)chain) getLogger) AnyBinaryIODictIterableListOptionalSetTupleUnion)urlparse)config) LicenseCLN) file_hashretry_onrun_with_umask) rate_limitHOUR) to_thread) default_hookeulasigszrealtime-av-confz/var/imunify360/filesz$https://files.imunify360.com/static/i g?ceZdZdZdS)IntegrityErrorzERaised when on disk content does not match hashes in description.jsonN__name__ __module__ __qualname____doc__S/opt/imunify360/venv/lib/python3.11/site-packages/defence360agent/files/__init__.pyr%r%RsOOOOr,r%ceZdZdZdS) UpdateErrora Raised on other errors during files update. Possible reasons are: * server returns non 200 status; * hash mismatched between downloaded content and description.json; * urllib errors; * JSON decoding errors; * errors while writing to disk. Nr&r+r,r-r/r/Vs    r,r/cKtd||tjt jd|ztzd{VdS)Nz2Files update failed with error: {err}, try: {try_})errtry_r)loggerwarningformatasynciosleeprandom randrange_TIMEOUT_MULTIPLICATOR)excis r-_log_failed_updater=csy NN<CC! D   -(a003II J JJJJJJJJJJr,pathmodereturnctd5tj|tjtjztjz|}dddn #1swxYwYtj|dS)zbOpen file at `path` using permission `mode` for writing in binary mode and return file object.rNwb)rosopenO_WRONLYO_CREATO_TRUNCfdopen)r>r?fds r-_open_with_moderJms   HH WT2;3bj@$ G GHHHHHHHHHHHHHHH 9R  s;AAAc t||5}tjtj|d|ddcdddS#1swxYwYdS)Ntimeoutfileheaderszutf-8)encoding) _fetch_urljsonloadio TextIOWrapperget_content_charset)urlrMresponses r-_fetch_json_syncrYus C ) ) ) Xy   !),@@II                       sAA&&A*-A*)on_error max_triesrWcKtj} |dt||d{VS#tt jf$r(}td||d}~wtj $r#td|t$r#td|t$r}td|d|d}~wtjjt jjf$r(}td||d}~wt&$r}td|d |d}~wwxYw) zDownload and decode JSON from *url*. Return decoded JSON. Raise UpdateError: * HTTP response status code is not 200; * Unicode or JSON decoding fails; * on time outs during HTTP request; * on other HTTP errors. Nz!json decode error [{}] for url {}request to {} timed outrequest to {} reset%eof error while updating files, url: , err: 8urllib/http error while updating files, url: {}, err: {} Can't fetch , reason: )r6get_event_looprun_in_executorrYUnicodeDecodeErrorrRJSONDecodeErrorr/r5socketrMConnectionResetErrorEOFErrorhttpclient HTTPExceptionurlliberrorURLErrorOSError)rWrMloopes r- _fetch_jsonrts  ! # #D=))$0@#wOOOOOOOOO  4 5NNN=DDQLLMMM >AAA3::3??@@@ ===/66s;;<<<     CC C C C C    K %v|'< =    F M MQ     ===;;;;;<<<=s:":E"#A33A(E"C00(E"#D;; E"EE"rOrMctj|dtjpdi|d}tj||5}|j|jfcdddS#1swxYwYdS)z>Perform HEAD http request to *url* with *timeout* & *headers*.Imunify-Server-IdHEAD)rOmethodrLN)rnrequestRequestr get_server_idurlopencoderO)rWrMrOreqrs r-_perform_http_head_syncrs .  !9!;!;!Ar   !  C   W  5 5!vqy !!!!!!!!!!!!!!!!!!sA66A:=A: current_mtimecRK|turdSt|d} tt||d|id{V\}}|dkrt d|d|t t 5t|d }||kr$t d ||d ||dddn #1swxYwYdS#tj $r#t d |t$r#t d |tjjt$jjf$rI}t+|d r|jdkrYd}~dSt d ||d}~wwxYw)zCheck if we need to download description.json file: - perform HEAD request if local file exists and older return True otherwise return False T)usegmtzIf-Modified-SinceruNzUnexpected http code z for z Last-ModifiedaGot code %r, but last modification date %s is earlier than or equal to the date provided in the If-Modified-Since header, the origin server SHOULD generate a 304 (Not Modified) response [rfc7232]. Here's curl cmd: curl -s -I -w '%%{http_code}' -H 'If-Modified-Since: %s' '%s'r]r^ri0Fra)_NEVERrrrr/r Exceptionr timestampr3r4rhrMr5rirkrlrmrnrorphasattrr)rWrrMformatted_mtimerrO last_mtimerss r-_need_to_downloadrsWt t<<AAA3::3??@@@ ===/66s;;<<< K %v|'< =   1f   !&C--55555 F M MQ      s1#C2ACCCBF&F!>#F!!F&T)compressc#Ki}|||d<dtjpdi}|r|dditj||}tjj|fi|5}t5}|j ddk}|rl|sj|j d d krLt d |j d|j ||r#| t| n||jd Vdddn #1swxYwYddddS#1swxYwYdS)zs Fetch *url* as binary file. If *compress* is true, ungzipping is done automatically if necessary. NrMrwrxzAccept-EncodinggzipruContent-Encodingz Content-Typezapplication/zipzRequested gzip but got Content-Encoding=%r. Read response as is [identity]. Headers: %s, as curl cmd: curl -Is -H 'Accept-Encoding: gzip' '%s')fileobj)rNrO)rr}updaternr{r|r~rrOgetr3infoitems enter_contextr ) rWrMr parameters req_headersrrXstackgzippeds r-rQrQs1J ' 9& (@(B(B(HbIK8-v6777 . k : :C        9;; "'"&&'9::fD    $$^448III KKJ $$%788 &&((    ##HX$>$>$>???'      %                                 s74E%B>E  E% E E%E E%%E),E) dest_filec0tj}|}t|||5}|dt x}rL|||||dt x}Ldddn #1swxYwY|dddks||z } |ddd} | Nt| } | | kr9tj d | | | z d |} || |krtd |d |d | | S)z Fetch *url* to *dest_file* and return its md5sum. Raise *urllib.error.ContentTooShortError* if the downloaded file has unexpected length. )rMrrNNrOrrzContent-Lengthz&{got} bytes read, {diff} more expected)gotdiff)messagecontentzcontent fetched from z does not match hash: expected=z, got=)hashlibmd5tellrQread_BUFSIZErwriterintrnroContentTooShortErrorr5 hexdigestr/) rWrrMrmd5sumrinitial_file_offsetrXchunk file_lengthcontent_length_headerexpected_file_length got_md5sums r-_fetch_n_md5sum_urlrs +--C#..** C8 < < <#',,X666e # JJu    OOE " " " ',,X666e ################ I  " "#5 6 6& @ @ nn&&)<< !) 3 7 78H$ O O ,#&'<#=#= #{22l77DKK'1K?L! 8J jF22 4C 4 4 4 4'1 4 4    sA/B55B9<B9rr dest_path dest_modec K t||5}tt|||||d{VcdddS#1swxYwYdS#tj$r#t d|t$r#t d|t$r}t d|d|d}~wtj j tj jf$r(}t d||d}~wt$r}t d|d |d |d}~wwxYw) zFetch bytes from `url`, save them to `dest_path`, and return md5 checksum of downloaded content. Raise UpdateError: * HTTP response status code is not 200; * on time outs during HTTP request; * on other HTTP errors. rNr]r^r_r`rarbz to rc)rJrrrhrMr/r5rirjrkrlrmrnrorprq)rWrrMrrrrrss r-_fetch_and_saverCs*L Y 2 2 i"#!                    >AAA3::3??@@@ ===/66s;;<<<     CC C C C C    K %v|'< =    F M MQ     LLLJJJ)JJqJJKKKLsRA A AAAA AA%D=3C(D=0#D D= D88D=_Itemrdatac&d|dDS)z,Return a set of _Item for easy manipulation.cFh|]}t|d|dS)rWr)r).0items r- z_items..ys* I I I4E$u+tH~ . . I I Ir,rr+)rs r-_itemsrws I I4= I I IIr,cd}|||tj|D]d\}}}|D],}|tj|||-|D],}|tj|||-edS)zCheck and change file/dir modes recursively. Starting at dirname, change all inner directory permissions to dir_perm, file permissions to file_perm c tj|jdz}||krmtj|sPt d|t|t|tj||dSdSdS#t$rt d|YdSwxYw)NizGFixing wrong permission to file/dir %s [%s] expected [%s] (not symlink)z&Failed to change permission to file %s) rClstatst_moder>islinkr3r4octchmodPermissionErrorro) file_dir_path permission current_modes r- _os_chmodz"check_mode_dirs.._os_chmods 8M22:UBLz))"'..33);! %% OO  33333*)))    LL8-       sB B%B>=B>N)rCwalkr>join) dirnamedir_perm file_permrr>dirsfiles directorynames r-check_mode_dirsr|s&Igx   WW--;;dE ? ?I Ibgll433X > > > > ; ;D Ibgll4.. : : : : ;;;r,description_path files_pathc||jvsJ|j}|}||krR|r/|d|dddS|j}||kPdSdS)aP Try to fix the structure of /var/imunify360/files/ when NotADirectoryError happens. It indicates that some part in the path is a file: /var/imunify360/files/sigs <- is a file => open("/var/imunify360/files/sigs/v1/description.json") will fail. We try to rectify it by deleting the file but up to FILES_DIR. T) missing_ok)parentsexist_okN)rparentis_fileunlinkmkdir)rr_dir topmost_dirs r-_fix_directory_structurers )1 1 1 1 1  "DK *   <<>>  KK4K ( ( (   dT  : : : E{ *      r,c<eZdZeejZeeZiZ iZ eZ eZ iZ dZedezejZd=dZdZdZdejd d fd Zdejfd Zd Zeddddedededededed d fdZ ed efdZ!ed e"efdZ#eded efdZ$dZ%d e"efdZ&edZ'd e(efdZ)dZ*e+fd e,fdZ-d efd Z.d!e,d efd"Z/d>d#Z0de1j2d$e"e3d d fd%Z4d&e"e3d e5e"e3e"effd'Z6eded efd(Z7eded efd)Z8e9d>d*Z:e9d+e1j2d e1j2fd,Z;d efd-Ze9d1ejd2ejd3e"ed d fd4Z?d5ed efd6Z@d?d7ZAd?d8ZBd>d?d9ZCe d@d:eDed d fd;ZEeded d fd<ZFd S)AIndexz/static)periodTcX||jvrtd|d|j||_d|_dgi|_|} t |5}tj||_dddn #1swxYwYn#t$rEt d|ttj|tYnTt t"tjf$r6}|r#t'd||d|_Yd}~nd}~wwxYw|rX|}t-|r5t'd d ||js|dSdS) z :param bool integrity_check: check if last update did not break anything (by interrupting it in the middle or another programmatic error) :raise IntegrityError: z*Trying to initiate unregistered file type z. Allowed types FrNzPath %s has a file in parentszcannot read description file {}Tz'some files are missing or corrupted: {}z, )_TYPES ValueErrortype _is_blank_json_descriptionfile_pathrDrRrSNotADirectoryErrorr_throttled_log_errorrpathlibPath FILES_DIRFileNotFoundErrorrfrgr%r5_corrupted_fileslenrr)selftype_integrity_checkr>frs bad_filess r-__init__zIndex.__init__s)  # #(U((+((  r] ))++ "d *q!Yq\\  * * * * * * * * * * * * * * *! D D D  & &'F M M M $W\$%7%7 C C C C C     " " "  $5<}>)rr'rrrrrs r-__repr__zIndex.__repr__s\ '       4::<<((    r,rr@Nctd|j|||}||jddS)zjWhether *files_path* dir may be used for this type's file group. :raises: IntegrityError zValidating [%s]: %sTrN)r3rr_make_file_grouprr FileGroups r-validatezIndex.validatesT  )49jAAA))     $)T222222r,c6Gfddj}|S)zV Return FileGroup class: Index class with local path == *files_path*. c6eZdZededeffd ZdS))Index._make_file_group..FileGrouprr@cF|jksJtjSz+Return local base path for given file type.)rrCfspath)clsrrrs r-rz4Index._make_file_group..FileGroup.files_paths( ))))y,,,r,N)r'r(r) classmethodstrr)rrsr-r r  sP  -s -s - - - - - -[ - - -r,r )rrs`` r-rzIndex._make_file_groupsG  - - - - - - - - - - -r,c .tj|j}ttjtjttj |jtj |d|ddS)NdirrN) r_PERMSrrrCr>normpathrr_PATHSpardir)rpermss r-rzIndex.check_mode_dirssq TY' G   Y TY(?KK   %L &M      r,F)all_zip essentialr relative_pathrrrrc|j||r|j|||j|<||d|j|<||j|<dS)aAdd a type to known file types. * relative_path is a relative path to all files for that type. * dir_perm is permission mask used to create directories. * file_perm is permission mask used to create files. * all_zip is a flag which shows whether that type of files can be downloaded in all.zip archive. all.zip is expected to be on the server. * essential is whether the agent can start if there are errors updating that type. )rrNN)radd_ESSENTIAL_TYPESrr_ALL_ZIP_SUPPORT)rrrrrrrs r-add_typezIndex.add_type sj, u  ,  $ $U + + +) 5$,i@@ 5&-U###r,cBKtd|jDS)zuWhether essential files exist. Note: the files may be corrupted (integrity check is not performed). c3DK|]}t|dj VdS)FrN)rr)rrs r- z.Index.essential_files_exist..DsI  eU333= =      r,)allr rs r-essential_files_existzIndex.essential_files_exist=s9  -      r,c4|jS)z&Return a set of all known files types.)rcopyr's r-typesz Index.typesIsz   r,cbtjt|j|Sr)rCr>rrrrrs r-rzIndex.files_pathNs!w||Isz%'8999r,crtj||jdS)z9Return local path for description.json for current index.description.json)rCr>rrrrs r-rzIndex._descriptionfile_pathSs'w||DOODI668JKKKr,cRt}t|jD]}||j} t |t jt}n%#t$r| |Y_wxYw||j kr| ||S)z9Return a set of file paths that are missing or corrupted.) setrr localfilepathrWrrrrrrr)rrrr>actuals r-rzIndex._corrupted_filesWsEE 4:&& $ $D%%dh//D "4h??$    d### $$ d###s A!!BBc|j|S)z` usage example: >> async with Index.locked(WHITELISTS): ... )_lockr-s r-lockedz Index.lockedesyr,cDfdtjDS)z(Return iterable over all files in index.c3LK|]}|jVdSrr2rWrrrs r-r%zIndex.files..ps3LL""48,,LLLLLLr,)rrrs`r-rz Index.filesns'LLLL 9K9KLLLLr,c|jdS)z+Return 'items' field from JSON description.r)rrs r-rz Index.itemsrsz'""r,c~ tj|jS#t$r|cYSwxYw)zBReturn mtime of description file if it exists, otherwise -math.inf)rCstatrst_mtimerq)rdefaults r-_descriptionfile_mtimezIndex._descriptionfile_mtimevsJ 74557788A A   NNN s *- <<c|}|sdS|tjjzt jkS)z4Return True if last update was too late in the past.T)r@r FilesUpdatePERIODtime)r _desc_mtimes r- _is_outdatedzIndex._is_outdated}s<1133  4V/66DDr,rMcK|jpyt|dkpT|o@t ||j||d{VS)z>Return True if update from server is needed for current index.rN)rrrrFr_descriptionfile_urlrr@)rrMs r-is_update_neededzIndex.is_update_neededs N 4((**++a/ !!##+--di88//11 r,c td5tj|||ddddS#1swxYwYdS#t$r"}t t ||d}~wwxYw)z)Create local directory for current index.r)r?rN)rrCmakedirsrqr/r)rrdir_moderrss r- _makedirszIndex._makedirss -"" G G G(XFFFF G G G G G G G G G G G G G G G G G G - - -c!ff%%1 , -s2A6 A:A:A A/ A**A/ to_updatecK||}||jd}|j|jd}|j|jd}|D]}||j} t j| } t j| s| | |dt|j| |||j d{VdS)zX Fetch files from *to_update* set, verify hashes, save to *files_path*. FrrrNr)rrN) rrrr2rWrCr>risdirrMrr) rrrNrMr fgrL file_moderfilenamers r- _update_fileszIndex._update_filess ))   Yty% 8 8 89RW%e,Ibg&v.   D''11Hgooh//G7==)) Bw5AAA!#{           r, remote_itemsctj}fd|D}fd|D}||z }fd|D}||z }||fS)zFigure out what should be updated based on current items, file system state and remote items. Return tuple of files to fetch and files to delete. Files to fetch is a set of _Item. Files to delete is a set of file paths.cDh|]}|jSr+r9r:s r-rz+Index._calculate_changes..s)LLLt))$(33LLLr,cDh|]}|jSr+r9r:s r-rz+Index._calculate_changes..s)NNN**4844NNNr,cLh|] }|jv|!Sr+r9)rrrrs r-rz+Index._calculate_changes..s>   !!$(++9<< <<J#>???r,c xKtj||j}t|}||jdz}|j|jd}|j|jd}| |j}t5}| ||d| tj ||dt||||dd {V} tj|d 5} | |d d d n #1swxYwYt%j|D]v\} } } | D]5}t%jt$j| ||6| D]5}t%jt$j| ||6w||t||}nR#t2t4t6tjtjf$r"}t=t?||d }~wwxYw| d d d n #1swxYwYt ||tC|dS) a Update current type of files using all.zip archive. Directory with current type of files will be cleared and replaced with all.zip contents. all.zip is expected to be on the server Return whether updated. :param timeout: :raise UpdateError: if OSError or http error or integrity check error (got wrong data from the server) all.ziprNrFrPT)rl)rrNr)"rrrrrrvrtrrrerrMcallbackrmrzipfileZipFile extractallrCrrr>rr _replace_live_with_new_dirrjr%rq BadZipfile LargeZipFiler/rpop_allbool)rrMrnnew_path archive_pathrSrL all_zip_urlrollback_stack_archiveroot directories filenamesrrTold_pathrss r-_run_update_all_zipzIndex._run_update_all_zipsNL!;!;<< ++I66))(-)*CDD K *62 ;ty)%0'' 22 [[2 %N NN8XN > > >  # #&! $   &# A 1_\37717&&x000111111111111111 57GH4E4EJJ0D+y%0JJ dI!>!>IIII$-JJdH!=!=yIIIIJ h'''!;;i"$  1 1 1"#a&&))q0 1  " " $ $ $e2 %2 %2 %2 %2 %2 %2 %2 %2 %2 %2 %2 %2 %2 %2 %j  lh    ts\>AJH*E  H E HE B>HJ-I"II""JJ  J rc ||jdz}d}t5}||d||j|r|dnd}tdD]} | |n#t$r|r|sj||jdz}t d |||| |||j |YwxYw|tj|d |dddn #1swxYwY|S) zbReplace *live_path* with *new_path*. Return *old_path* :raises: OSError liveNT)target_is_directoryFstrictz .live-movedz1Moving %s [live] to %s, to rename %s to it [live]rg)rtrr symlink_toryr is_symlinkresolverangerenameIsADirectoryErrorr3rreplacerirjr)rrn new_live_path moved_pathrrlasts r-r}z Index._replace_live_with_new_dir:s!**8=6+ABB  [[- %N  $ $X4 $ H H H  # #M$8 9 9 9'')) !!!/// a O OO!((333E(OOO %//11O%2%<%<).>&&  9%&) "))*555'// 0BINNN-O0% j====  " " $ $ $[- %- %- %- %- %- %- %- %- %- %- %- %- %- %- %^s7A/E>B53E>5B EE>E/E>>FFc Kj}t||d{V}t |\}}|p|}|s6t djdStj j}| r| dnd}t|} t5} | jjdd| t&j| d |r|r|n|} | rAt| | |fd |Dd{V| ||d{V t3| d z jjd 5} | t7j|dddn #1swxYwY| | |}n6#t@tBf$r"} tEtG| | d} ~ wwxYw| $dddn #1swxYwY|rE|r1t d |t'j|d dS)z Run update, return whether updated. :raise UpdateError: if OSError or http error or integrity check error (got wrong data from the server) rLNzupdating %s: nothing to update.FrrrPTrgc3LK|]}|jVdSrr9r:s r-r%z$Index._run_update..sD$$9=**4844$$$$$$r,r/rNz,Removing old path on file by file update: %s)%rHrrtr`rr3r_touchrrrrrrrvrrMrryrirjis_dir _copytreeunionrUrJrrRdumpsencoder r}r%rqr/rr)rrMrWas_jsonrNr^ need_updaternrrr from_pathrNrss` r- _run_updatezIndex._run_updatews '' 22#C999999999#66vgGG 9,9   KK949 E E E KKMMM5L!;!;<< 09/C/C/E/E OI  U  + + +4 ++I66[[* %N NN$+di07%      # # xt $   %I):):I  !! ooOO$$$$AJ$$$$$Xy'$JJ J J J J J J J 1$11K *62=JJtz'2299;;<<< =============== h''' ::8YOO"G, 1 1 1!#a&&))q0 1  " " $ $ $U* %* %* %* %* %* %* %* %* %* %* %* %* %* %* %Z  8)) 8 KK>    M($ 7 7 7 7ts[CK15)J:I$ J$I( (J+I( ,.JK1K,K  KK11K58K5from_dirto_dir ignored_pathsc`Kfd}ttj||d|dd{VdS)z7Copy *from_dir* to *to_dir* except for *ignored_paths*.cttjtsJt fd|DS)z(Return names that should not be copied.c3`K|](}tj|v$|V)dSr)rCr>r)rrrr>s r-r%z8Index._copytree..ignore_names..sJ7<<d++}<<<<<<r,) isinstancerCrr frozenset)r>namesrs` r- ignore_namesz%Index._copytree..ignore_namess_bioos33 3 33! r,T)symlinksignore dirs_exist_okN)rricopytree)rrrrs ` r-rzIndex._copytreesu       O               r,rWctjt|j|j}|j|j}tj|tj|j vsJd ||tj||}tj | |j|S)z.Return a local file path corresponding to URL.z$url ({}) does not fit file path ({})) rCr>relpathr_URL_PATH_PREFIXrrrrrr5rr)rrW url_relpath type_pathrs r-r2zIndex.localfilepathsgoo SMM  5  K * L # #w|K'@'@'H H H H 1 8 8i H H I H H Y?? w||DOODI66 FFFr,c |}tj|rtj|dSdS#t $r2}t t|Yd}~dSd}~wwxYw)z5Update mtime of description.json file so it is fresh.N) rrCr>isfileutimerqr3r4r)rr>rss r-rz Index._touchs #--//Dw~~d##    # # # NN3q66 " " " " " " " " " #sAA B 'BB c2Kt|j|jtgD]H} |||d{V#t$r&}t d||Yd}~Ad}~wwxYwt d|jd| zdS)Nzhook %s error: %sz%s files update finished%sz (not updated))r _HOOKSrr rr3 exceptionr)r is_updatedhookrss r- _run_hookszIndex._run_hookss$+di0<.AA ? ?D ?d4,,,,,,,,,, ? ? ?  !4dA>>>>>>>> ? ( I J /     s? A/ A**A/c$Ktjj}|sy||d{Vs^td|jttjjdz| dd{VdS|j o|j |j}| }|rd}td|j| tj |tjj|d{V}|r!td|j|nG#tjt"f$r.}td |j||d }Yd}~nd}~wwxYw|rd }td|j| tj |tjj|d{V}|r!td|j|nr#tjt"f$rY}td |j||| dd{V|j|jvr|Yd}~dSd}~wwxYw| |p|d{VdS) aRun update for the current `type` of files. Normally update is performed when either is true: * index is never been fetched (description.json missing or broken); * last update was performed longer than configured period of time ago; * some local files are missing or have wrong content (md5 hash differs from description.json). If force is True then update is performed unconditionally. Raises asyncio.TimeoutError, UpdateError. Nz(%s was updated less than %s minutes ago.<F)rrxzUpdating %s files via %szUpdated %s using %sz%s update error via %s: %sTzfile by file download)rrBTIMEOUTrIr3rrrrCrrr!r6wait_forrSOCKET_TIMEOUT TimeoutErrorr/rorr )rforcerMr file_by_filelog_strupdatedrss r-rz Index.updates$, 4#8#8#A#AAAAAAA  KK: F&-344    //U/33 3 3 3 3 3 3 3 F.ET%:49%E"{  $G KK2DIw G G G $ ' 0,,*9 !! KKK 5ty'JJJ(+6 $ $ $ 0$)Wa $  $  -G KK2DIw G G G  ' 0$$V%7%FGG!!KKK 5ty'JJJ(+6    0$)Waooo7777777779 555GFFFFF oo)9Eo:::::::::::s3A D22E6$E11E6A HI/AI**I/ only_typecK|rj||d}||4d{V||d{Vdddd{VdS#1d{VswxYwYdS|rtd|jD]i}||d}||4d{V||d{Vdddd{Vn#1d{VswxYwYjdStd|jD]i}||d}||4d{V||d{Vdddd{Vn#1d{VswxYwYjdS)zkRun update for all registered `types` of files. Raises asyncio.TimeoutError, UpdateError. FrNzUpdating essential fileszUpdating all files)r6rr3rr r)rrronly_essentialindexrs r- update_allzIndex.update_allDss  .C 5999Ezz),, * * * * * * * *ll5))))))))) * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *  . KK2 3 3 3- . .E5999::e,,........,,u---------........................... . . KK, - - - . .E5999::e,,........,,u---------........................... . .s5A A&)A&<C** C4 7C4 E77 F F cF|j||dS)z:Add a hook for type_ to be called after successful update.N)rr)rrrs r-add_hookzIndex.add_hook]s% 5d#####r,)T)Fr@N)NFF)Gr'r(r)rr6Lockr5r1rrrrr r!rrrr3rorrrrrCPathLiker rrrrrrr"r(rr+rrrr6rrrrfloatr@rFrIrMrrrrUrr`rHre staticmethodrmrvrr}rrr2rrrrrrr+r,r-rrs K % %E [  F F F SUUFsuu 6::QX666v|DD)#)#)#)#V       32; 34 3 3 3 3 2;       ..... .  ... ...[.8  D    [  !c#h!!![!:s:s:::[:LLL #c(      [ Mx}MMMM###.4EdEEEE  e       ----!,36u: 2$J$ s5z3s8# $$$$$.KKKKK[KBBBBB[B : : :\ :@gl@w|@@@\@LDLLLL\:,:+2<: :::\:xMDMMMM^ + '){ CFs8    \ . G G G G G G####     B;B;B;B;B;HJO..  . ...[.0$S$4$$$[$$$r,rcttddddttdddd ttd dddd S) zRegister required file types.zeula/v1iiF)rzsigs/v1iiTzrealtime-av-conf/v1N)rr"EULASIGSREALTIME_AV_CONFr+r,r- configurercsn NN4E5%N@@@ NN4E5$N??? NN   r,FrcK t||d{VS#tjtf$r'}t d||Yd}~dSd}~wwxYw)z.Run files.update and log Update/TimeoutErrors.Nz*Failed to update files [%s] with error: %s)rrr6rr/r3ro)rrr1s r-update_and_log_errorrts %%i777777777  + .    8)S          s %A"AA"cNK tdd{VS#tjtf$ri}td{Vrt d|n#t|tjrt|Yd}~dSd}~wwxYw)z6Update all files. Don't fail if essential files exist.T)rNz2Failed to update files [essential files exist]: %s) rrr6rr/r(r3ror)r1s r-!update_all_no_fail_if_files_existrs %%T%:::::::::  + .   ,,.. . . . . . .  LLDc    #w344 !s*       s %B$ABB$r)NF)ir*r6rqrpr http.clientrkrTrRmathrCrr8rirhrDrz urllib.errorrnurllib.request collectionsrr contextlibrrr email.utilsrr rr itertoolsr loggingr typingr rrrrrrrr urllib.parserdefence360agent.contractsr!defence360agent.contracts.licenserdefence360agent.utilsrrrdefence360agent.utils.commonrrdefence360agent.utils.threadsrhooksr r'r3rrrrrrcr_MAX_TRIES_FOR_DOWNLOADr:infrrrrrJSONType RuntimeErrorr%r/r=rrJrYrtrrrQrrrrrrrrrrr(rrr+r,r-rsP  ////////::::::::::99999999                      "!!!!!-,,,,,888888EEEEEEEEEE99999999333333 8   % GL0 1 1 1  ( c5$d38nd3iG HPPPPP\PPP     ,   KKK"+SX h     ,8O=3=H====D*, ! ! !  ! ! ! ! !  ,8O4 4"4-24 4444n59) ) ) C) U) ) ) ) X&&&&&R ,8O +L+L+L +L{+L  +L +L+L+L+L\  7UH-..JJUJJJJ ;;;F@Il07  ,l $l $l $l $l $l $l $l $^      3,1   }             r,