o
    RDiJ                     @  s  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlmZm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZmZmZmZ d dlZd
dlmZmZmZmZmZ d
dl m!Z!m"Z" d
dl#m$Z$ d
dl%m&Z& d
dl'm(Z(m)Z) d
dl*m+Z+ d
dl,m-Z-m.Z. d
dl/m0Z0 erd
dl1m2Z2m3Z3 ee4Z5G dd deZ6e(j7e(j8e(j9e(j:e(j;dZ<dddZ=G dd deZ>dS )     )annotationsN)ABCabstractmethod)defaultdict)BytesIO)	getLogger)ceil)Path)TYPE_CHECKINGAnyCallable
NamedTuple   )HTTP_HEADER_CONTENT_ENCODINGREQUEST_CONNECTION_TIMEOUTREQUEST_READ_TIMEOUT
FileHeaderResultStatus)EncryptionMetadataSnowflakeEncryptionUtil)RequestExceedMaxRetryError)SnowflakeFileUtil)SessionManagerSessionManagerFactory)requests)ConnectionErrorTimeout)HTTPResponse)SnowflakeFileMetaStorageCredentialc                   @  s&   e Zd ZU ded< ded< ded< dS )SnowflakeFileEncryptionMaterialstrquery_stage_master_keyquery_idintsmk_idN)__name__
__module____qualname____annotations__ r*   r*   [/var/www/Datamplify/venv/lib/python3.10/site-packages/snowflake/connector/storage_client.pyr    '   s   
 r    )GETPUTPOSTHEADDELETEresprequests.ResponsereturnNonec                 K  s8   t | jv rt| jtrd| j_| jjt  dS dS dS )zLRemove content-encoding header and decoder so decompression is not triggeredN)r   headers
isinstancerawr   _decoderpop)r1   kwargsr*   r*   r+   remove_content_encoding6   s   
r;   c                   @  s.  e Zd ZdZejjeefZ	dZ
dZ				dTdUddZdVddZdVddZdVddZedWd"d#ZdVd$d%ZdVd&d'ZdVd(d)ZedXd,d-ZdYd2d3Zd4d5 ZdVd6d7ZdZd;d<ZdVd=d>Zd[d?d@Zed\dBdCZed[dDdEZdXdFdGZdVdHdIZdVdJdKZ dVdLdMZ!dVdNdOZ"dVdPdQZ#dVdRdSZ$dS )]SnowflakeStorageClient)i  i  i  i  i  i  g      0@g      ?TN   Fmetar   
stage_infodict[str, Any]
chunk_sizer$   chunked_transferbool | NonecredentialsStorageCredential | None	max_retryunsafe_file_writeboolr3   r4   c                 C  s   || _ || _tt| _t | _d | _d | _	|| _
|| _|j|_|j|_d| _| j jr:tj| j jtj| j jnd | _| j jrHt| jd nd | _|| _|| _d| _t | _d| _ d| _!d| _"|| _#d S )NFz.partr   )$r>   r?   r   r$   retry_counttempfilemkdtemptmp_dir	data_fileencryption_metadatarF   rD   src_file_namereal_src_file_namesrc_file_sizeupload_sizepreprocessedlocal_locationospathjoinbasenamedst_file_namefull_dst_file_namer	   intermediate_dst_pathrB   rA   num_of_chunks	threadingLocklocksuccessful_transfersfailed_transferslast_err_is_presigned_urlrG   )selfr>   r?   rA   rB   rD   rF   rG   r*   r*   r+   __init__E   s<   


	

zSnowflakeStorageClient.__init__c                 C  sZ   | j jr+| j }td|j  |jrt|j\|_}d S t	|j| j
\|_}d S d S )Nzcompressing file=)r>   require_compressloggerdebugrO   intermediate_streamr   compress_with_gzip_from_stream
src_streamcompress_file_with_gziprL   rP   )rc   r>   rR   r*   r*   r+   compressv   s"   zSnowflakeStorageClient.compressc                 C  sX   | j }td|j  |jd u rt|j\|_|_d S t	|j
p$|j\|_|_d S )Nzgetting digest file=)r>   rf   rg   rP   rh   r   get_digest_and_size_for_filesha256_digestrR   get_digest_and_size_for_streamrj   rc   r>   r*   r*   r+   
get_digest   s   


z!SnowflakeStorageClient.get_digestc                 C  s   | j }td|j  |jd u r*tj|j|j| jd\| _	| _
tj| j
|_d S t }|jp2|j}|d t|j||| _	|d |dtj|_|d |jd ur]|j  ||_|j| _
d S )Nzencrypting file=)rL   r   )r>   rf   rg   rP   rh   r   encrypt_fileencryption_materialrL   rN   rM   rU   rV   getsizerR   r   rj   seekencrypt_streamSEEK_ENDclose)rc   r>   encrypted_streamrj   r*   r*   r+   encrypt   s2   





zSnowflakeStorageClient.encryptfilenamer!   FileHeader | Nonec                 C     dS )zCheck if file exists in target location and obtain file metadata if exists.

        Notes:
            Updates meta.result_status.
        Nr*   )rc   r{   r*   r*   r+   get_file_header   s   z&SnowflakeStorageClient.get_file_headerc                 C  s   | j }td|j  | |j}|js;|   |jt	j
kr;td| jd  d|j d d|_t	j|_d| _d S |jrB|   |   |jr_|r_|j|jkr_td|j d	 t	j|_d| _d S )
NzPreprocessing zfile already exists location="locationz", file_name=""r   Tzsame file contents for z, skipping upload)r>   rf   rg   rO   r~   rY   	overwriterq   result_statusr   UPLOADEDr?   dst_file_sizeSKIPPEDrS   re   rl   skip_upload_on_content_matchrn   digestname)rc   r>   file_headerr*   r*   r+   
preprocess   s8   
z!SnowflakeStorageClient.preprocessc                 C  s   | j }| js|   n	|jrt| j td|j	  |jr%| 
  n|j| _td |j|jk s7| js;d| _n	t|j| j | _td| j  i | _t| jD ]}d| j|< qU| jrk| jdkrm|   d S d S d S )NzPreparing to upload zfinished preprocessingr   znumber of chunks r   )r>   rS   r   rs   rU   removerM   rf   rg   rO   rz   rP   rR   multipart_thresholdrB   r\   r   rA   rI   range_initiate_multipart_upload)rc   r>   chunk_idr*   r*   r+   prepare_upload   s(   


z%SnowflakeStorageClient.prepare_uploadc                 C  s   | j }| j| jkr*| jdkr*| jdkr|   tj|_|j|_t	
|j d d S d|_t	
|j d | jdkr?|   tj|_d S )Nr   r   z upload is completed.z upload is aborted.)r>   r`   r\   _complete_multipart_uploadr   r   r   rR   r   rf   rg   rO   _abort_multipart_uploadERRORrp   r*   r*   r+   finish_upload   s   

z$SnowflakeStorageClient.finish_uploadresponser2   c                 C     d S Nr*   rc   r   r*   r*   r+   _has_expired_token     z)SnowflakeStorageClient._has_expired_tokenverbget_request_args*Callable[[], tuple[bytes, dict[str, Any]]]retry_idc              
   C  sN  t | }d}d }| jjr| jjjjr| jjjj}| j| | jk rtd| j|   | j	j
}| \}}ttf|d< z|rh|jj|d}	td|	  |	j||fi |}
W d    n1 sbw   Y  ntd tjdd}|||fi |}
| |
rtd	 |   nJd| _|
j| jv rtd
|
j  ttd| j|  | j | j | j|  d7  < n| |
rtd | j	| | j|  d7  < n|
W S W n> | jy } z0d| _ttd| j|  | j | j t| d| d|  | j|  d7  < W Y d }~nd }~ww | j| | jk s!t | d| d)N    zretry #timeout)urlz$storage client request with session z'storage client request with new sessionF)use_poolingz6presigned url expired. trying to update presigned url.ztransient error:    r   z(token is expired. trying to update tokenz
 with url z failed for transient error: z& failed for exceeding maximum retries.)!METHODSr>   sfagent_cursor
connectionrI   rF   rf   rg   rD   	timestampr   r   restuse_sessionrequestr   get_manager_has_expired_presigned_url_update_presigned_urlrb   status_codeTRANSIENT_HTTP_ERRtimesleepmin
SLEEP_UNIT	SLEEP_MAXr   updateTRANSIENT_ERRORSwarningr   )rc   r   r   r   	rest_callr   conncur_timestamprest_kwargssessionr   session_managerer*   r*   r+   _send_request_with_retry  sv   





8z/SnowflakeStorageClient._send_request_with_retryc                 C  s$   | j  s| j jdd | j |S )Ni  )mode)r[   existstouchopen)rc   r   r*   r*   r+   _open_intermediate_dst_pathY  s   
z2SnowflakeStorageClient._open_intermediate_dst_pathc                 C  s   t j| j}t j|st | | | jj}|r"|j	r"|j	| _	d| _
|rC|jrC|j| j_| jrC| jj| jjkrCt|j| j | _
| d}|| jj W d    d S 1 s[w   Y  d S )Nr   zwb+)rU   rV   dirnamerZ   r   makedirsr~   r>   rP   rN   r\   content_lengthrQ   rB   r   r   rA   r   truncate)rc   base_dirr   fdr*   r*   r+   prepare_download^  s    



"z'SnowflakeStorageClient.prepare_downloadr   databytesc                 C  sJ   |  d}|| j|  || W d   dS 1 sw   Y  dS )zIWrites given data to the temp location starting at chunk_id * chunk_size.zrb+N)r   ru   rA   write)rc   r   r   r   r*   r*   r+   write_downloaded_chunkx  s   "z-SnowflakeStorageClient.write_downloaded_chunkc                 C  s  | j }| jdkrj| j| jkrjtj|_|jrKtd| j	  |j
d ur-| |j}|j| _tj| j|jt| j| j| jd}t|| j	 | j  ntd| j	  tt| j| j	 t| j	}|j|_d S tj| j	rwt| j	 td| j	  d|_tj|_d S )Nr   zencrypted data file=)rL   rG   znot encrypted data file=zFailed to download a file: )r>   r\   r`   r   
DOWNLOADEDr   rs   rf   rg   rZ   presigned_urlr~   rO   rN   r   decrypt_filer!   r[   rL   rG   shutilmoveunlinkrU   statst_sizer   rV   isfile	exceptionr   )rc   r>   r   tmp_dst_file_name	stat_infor*   r*   r+   finish_download  s4   
	z&SnowflakeStorageClient.finish_downloadc                 C  s   t | jjp| jj }| jjp| jjpt| jd}z | jdkr$| }n||| j	  || j	}W |r9|
  n|rA|
  w w td| d| j  | || td| d| j  d S )Nrbr   zUploading chunk z	 of file zSuccessfully uploaded chunk )rH   r>   rj   rh   r   rM   r\   readru   rA   rx   rf   rg   _upload_chunk)rc   r   
new_streamr   _datar*   r*   r+   upload_chunk  s*   



z#SnowflakeStorageClient.upload_chunkchunkc                 C  r   r   r*   )rc   r   r   r*   r*   r+   r     r   z$SnowflakeStorageClient._upload_chunkc                 C  r   r   r*   )rc   r   r*   r*   r+   download_chunk  r   z%SnowflakeStorageClient.download_chunkc                 C  r}   )NFr*   r   r*   r*   r+   r        z1SnowflakeStorageClient._has_expired_presigned_urlc                 C  r   r   r*   rc   r*   r*   r+   r     r   z,SnowflakeStorageClient._update_presigned_urlc                 C  r   r   r*   r   r*   r*   r+   r     r   z1SnowflakeStorageClient._initiate_multipart_uploadc                 C  r   r   r*   r   r*   r*   r+   r     r   z1SnowflakeStorageClient._complete_multipart_uploadc                 C  r   r   r*   r   r*   r*   r+   r     r   z.SnowflakeStorageClient._abort_multipart_uploadc              
   C  s   t j| jr7td| j  zt| j W n ty6 } ztd| j d|  W Y d}~nd}~ww | j	j
rH| j	j
jsJ| j	j
  dS dS dS )zoDeletes the tmp_dir and closes the source stream belonging to this client.
        This function is idempotent.zcleaning up tmp dir: zFailed to delete z: N)rU   rV   r   rL   rf   rg   r   rmtreeOSErrorr>   rj   closedrx   )rc   exr*   r*   r+   delete_client_data  s   $z)SnowflakeStorageClient.delete_client_datac                 C  s   |    d S r   )r   r   r*   r*   r+   __del__  s   zSnowflakeStorageClient.__del__)TNr=   F)r>   r   r?   r@   rA   r$   rB   rC   rD   rE   rF   r$   rG   rH   r3   r4   )r3   r4   )r{   r!   r3   r|   )r   r2   r3   rH   )r   r!   r   r   r   r$   r3   r2   )r   r$   r   r   r3   r4   )r   r$   r3   r4   )r   r$   r   r   r3   r4   )%r&   r'   r(   r   OpenSSLSSLSysCallErrorr   r   r   r   r   rd   rl   rq   rz   r   r~   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r*   r*   r*   r+   r<   >   sJ    
1



"

H



(





r<   )r1   r2   r3   r4   )?
__future__r   rU   r   rJ   r]   r   abcr   r   collectionsr   ior   loggingr   mathr   pathlibr	   typingr
   r   r   r   r   	constantsr   r   r   r   r   encryption_utilr   r   errorsr   	file_utilr   r   r   r   vendoredr   vendored.requestsr   r   vendored.urllib3r   file_transfer_agentr   r   r&   rf   r    getputpostheaddeleter   r;   r<   r*   r*   r*   r+   <module>   sD    
	