o
    RDiZ                     @  sD  d dl mZ d dlZd dlZd dlm  mZ d dlmZm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZmZ d dlmZmZ d	d
lmZmZ d	dlmZmZmZmZ d	dlm Z  d	dl!m"Z"m#Z# d	dl$m%Z% erud	dl&m'Z'm(Z( ee)Z*dZ+dZ,dZ-dZ.dZ/dZ0dZ1dZ2dZ3e4dZ5G dd deZ6G dd de"Z7dS )    )annotationsN)datetimetimezone)IOBase)	getLogger)
itemgetter)TYPE_CHECKINGAny
NamedTuple)hasheshmac   )quoteurlparse)HTTP_HEADER_CONTENT_TYPEHTTP_HEADER_VALUE_OCTET_STREAM
FileHeaderResultStatus)EncryptionMetadata)SnowflakeStorageClientremove_content_encoding)requests)SnowflakeFileMetaStorageCredentialzx-amz-meta-z
sfc-digestzx-amz-matdescz	x-amz-keyzx-amz-iviE'  ExpiredTokenvirtualzUNSIGNED-PAYLOADz +c                   @  s   e Zd ZU ded< ded< dS )
S3Locationstrbucket_namepathN)__name__
__module____qualname____annotations__ r$   r$   ^/var/www/Datamplify/venv/lib/python3.10/site-packages/snowflake/connector/s3_storage_client.pyr   .   s   
 r   c                      s*  e Zd Z			dbdc fddZ	dddeddZedfddZedfddZedgddZedgd d!Z	edhd$d%Z
edid)d*Ze		+djdkd2d3Zedld9d:Zdmd=d>Zednd@dAZ						dodpdKdLZdqdOdPZdrdQdRZdsdSdTZdtdWdXZdsdYdZZdsd[d\Zdud]d^Zdvd`daZ  ZS )wSnowflakeS3RestClientNFmetar   credentialsr   
stage_infodict[str, Any]
chunk_sizeintuse_accelerate_endpointbool | Noneuse_s3_regional_urlboolunsafe_file_writereturnNonec                   s   t  j|||||d |d | _d| _d| _t| jd | _|p0d|v o(|d p0d|v o0|d | _	|
d| _d| _|d rLd	| jj d
|d  | _| | dS )zKRest client for S3 storage.

        Args:
            stage_info:
        )r(   r1   regionNlocationuseS3RegionalUrluseRegionalUrllocationTypeendPointhttps://.)super__init__region_name	upload_idetagsr&   _extract_bucket_name_and_pathr)   
s3locationr/   getlocation_typeendpointr   transfer_accelerate_config)selfr'   r(   r)   r+   r-   r/   r1   	__class__r$   r%   r=   4   s6   
	zSnowflakeS3RestClient.__init__c                 C  s   | j r| j drd| jj d| j  d| _dS | jd ur%| jddkS | jr7d| jj d| j  d| _dS |d u rUt| jj d	rNt	
d
 d}n| | jj}|rad| jj d| _n	d| jj d| _t	
d| j d |S )Nzcn-r:   z.s3.z.amazonaws.com.cnFzs3-accelerate.amazonaws.comr   z.amazonaws.comzsfc-zMNot attempting to get bucket transfer accelerate endpoint for internal stage.z.s3-accelerate.amazonaws.comz.s3.amazonaws.comzUsing z as storage endpoint.)r>   
startswithrB   r   rE   findr/   r   lowerloggerdebug_get_bucket_accelerate_config)rG   r-   r$   r$   r%   rF   f   s<   
z0SnowflakeS3RestClient.transfer_accelerate_config
secret_keybytes_inputr   c                 C  s(   t | t }||d | S )z5Applies HMAC-SHA-256 to given string with secret_key.utf-8)r   HMACr   SHA256updateencodefinalize)rP   rR   hr$   r$   r%   _sign_bytes   s   z!SnowflakeS3RestClient._sign_bytesc                 C  s   t t| |S )zJConvenience function, same as _sign_bytes, but returns result in hex form.)binasciihexlifyr&   rZ   )rP   rR   r$   r$   r%   _sign_bytes_hex   s   z%SnowflakeS3RestClient._sign_bytes_hexc                 C  s    t t  }||  | S )z$Applies SHA-256 hash to given bytes.)r   HashrU   rV   rX   )rR   digestr$   r$   r%   _hash_bytes   s   
z!SnowflakeS3RestClient._hash_bytesc                 C  s   t t| S )zJConvenience function, same as _hash_bytes, but returns result in hex form.)r[   r\   r&   r`   )rR   r$   r$   r%   _hash_bytes_hex   s   z%SnowflakeS3RestClient._hash_bytes_hexquery_partstuple[tuple[str, str], ...]c                 C  s   d dd | D S )zConvenience function to build the query part of a URL from key-value pairs.

        It filters out empty strings from the key, value pairs.
        &c                 S  s   g | ]
}d  tt|qS )=)joinfilterr0   ).0er$   r$   r%   
<listcomp>   s    zASnowflakeS3RestClient._construct_query_string.<locals>.<listcomp>rf   )rb   r$   r$   r%   _construct_query_string   s   z-SnowflakeS3RestClient._construct_query_stringheadersdict[str, str | list[str]]tuple[str, str]c              	     s   g }dd |   D  t  } fdd|D }|D ]%\}}t|tr*d|}|dd}|| d t	
d|   qd|}|rM|d7 }|d	|fS )
zConstruct canonical headers as per AWS specs, returns the signed headers too.

        Does not support sorting by values in case the keys are the same, don't send
        in duplicate keys, but this is not possible with a dictionary anyways.
        c                 S  s   i | ]	\}}|  |qS r$   )rL   rh   kvr$   r$   r%   
<dictcomp>   s    zUSnowflakeS3RestClient._construct_canonicalized_and_signed_headers.<locals>.<dictcomp>c                   s   g | ]}| | fqS r$   r$   )rh   rq   low_key_dictr$   r%   rj      s    zUSnowflakeS3RestClient._construct_canonicalized_and_signed_headers.<locals>.<listcomp>,
 :;)itemssortedkeys
isinstancelistrf   replaceappendstripRE_MULTIPLE_SPACESsub)rm   ressorted_headers_resrq   rr   ansr$   rt   r%   +_construct_canonicalized_and_signed_headers   s   	

$
zASnowflakeS3RestClient._construct_canonicalized_and_signed_headers verbcanonical_uri_parameterdict[str, str]canonical_headers!dict[str, str | list[str]] | Nonepayload_hashc                 C  sP   d dd t| tddD }t|\}}d | |pd||||g|fS )zBuild canonical request and also return signed headers.

        Note: this doesn't support sorting by values in case the same key is given
         more than once, but doing this is also not possible with a dictionary.
        rd   c                 s  s"    | ]\}}d  ||gV  qdS )re   Nrk   rp   r$   r$   r%   	<genexpr>   s    
zXSnowflakeS3RestClient._construct_canonical_request_and_signed_headers.<locals>.<genexpr>r   )keyrw   /)rf   r|   r{   r   r&   r   )r   r   rb   r   r   canonical_query_stringsigned_headersr$   r$   r%   /_construct_canonical_request_and_signed_headers   s(   

zESnowflakeS3RestClient._construct_canonical_request_and_signed_headersr>   service_nameamzdateshort_amzdatecanonical_request_hashc                 C  s2   | d|  d| d}d d|||dg|fS )a  Given all the necessary information construct a V4 string to sign.

        As per AWS specs it requires the scope, the hash of the canonical request and
        the current date in the following format: YYYYMMDDTHHMMSSZ where T and Z are
        constant characters.
        This function generates the scope from the amzdate (which is just the date
        portion of amzdate), region name and service we want to use (this is only s3
        in our case).
        r   z/aws4_requestrw   zAWS4-HMAC-SHA256rS   )rf   decode)r>   r   r   r   r   scoper$   r$   r%   _construct_string_to_sign   s   z/SnowflakeS3RestClient._construct_string_to_signresponserequests.Responsec                 C  s>   |j dkrdS |j}|r| rdS t|}|djtkS )aW  Extract error code and error message from the S3's error response.

        Expected format:
        https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#RESTErrorResponses

        Args:
            response: Rest error response in XML format

        Returns: True if the error response is caused by token expiration

        i  FCode)status_codetextisspaceET
fromstringrK   EXPIRED_TOKEN)rG   r   messageerrr$   r$   r%   _has_expired_token  s   

z(SnowflakeS3RestClient._has_expired_tokenr   c                 C  s2   |  d\}}}|r|ds|d7 }t||dS )Nr   )r   r   )	partitionendswithr   )stage_locationr   _r   r$   r$   r%   rA   '  s   z3SnowflakeS3RestClient._extract_bucket_name_and_pathurlretry_id	int | strdict[str, str] | Nonex_amz_headerspayload!bytes | bytearray | IOBase | Noneunsigned_payloadignore_content_encodingc
              
     s   d u ri  d u ri  d u rdd u ri t jjddd< jd< |r2td< nt 	 d< d f	d	d
}

|
|S )N    	AWS_TOKENr   x-amz-security-tokenhostx-amz-content-sha256r2   tuple[bytes, dict[str, bytes]]c                    st  t tjjd d} | d}|d d }|d< jjddd< j	j
jr1dj nd d	 d
\}}jd|||d \}}djjd  d|}|j}|d}	|	d}
|
| }ddjjd  d| d d| d d|d  }  | d< d i}r|d< rdti|d< d|fS )N)tzinfoz%Y%m%dT%H%M%SZ   z
x-amz-dater   r   r   rz   r   )r   r   rb   r   r   s3rS   AWS4AWS_SECRET_KEYaws4_requestzAWS4-HMAC-SHA256 zCredential=
AWS_KEY_IDr   z, zSignedHeaders=z
Signature=Authorizationrm   datar   hooks)r   nowr   utcr   strftimer(   credsrC   r   r   paramsr   r>   ra   rW   rL   rZ   r]   r   rV   r   )tr   r   canonical_requestr   string_to_signr   kDatekRegionkServicesigning_key	signatureauthorization_header	rest_args	rm   r   
parsed_urlr   rb   rG   r   r   r   r$   r%   &generate_authenticated_url_and_args_v4P  sd   
	

zqSnowflakeS3RestClient._send_request_with_authentication_and_retry.<locals>.generate_authenticated_url_and_args_v4)r2   r   )r   r(   r   rC   hostnameUNSIGNED_PAYLOADr&   ra   rL   r   _send_request_with_retry)rG   r   r   r   rb   r   rm   r   r   r   r   r$   r   r%   +_send_request_with_authentication_and_retry0  s*   

7zASnowflakeS3RestClient._send_request_with_authentication_and_retryfilenameFileHeader | Nonec                 C  s   t | jj|d }| jd|  }d}d| j|< | j|d|d}|jdkr_tj	| j
_|j}|tt rLt|tt |tt |tt dnd}t|tt t|d|d	S |jd
krxtd| jj d|  tj| j
_dS |  dS )zGets the metadata of file in specified location.

        Args:
            filename: Name of remote file.

        Returns:
            None if HEAD returns 404, otherwise a FileHeader instance populated
            with metadata
        r   HEADr   )r   r   r      )r   ivmatdescNzContent-Length)r_   content_lengthencryption_metadatai  znot found. bucket: z, path: )r   rB   r   lstriprE   retry_countr   r   r   UPLOADEDr'   result_statusrm   rC   META_PREFIXAMZ_KEYr   AMZ_IVAMZ_MATDESCr   
SFC_DIGESTr,   rM   rN   r   NOT_FOUND_FILEraise_for_status)rG   r   r   r   r   r   metadatar   r$   r$   r%   get_file_header  s<   



	

z%SnowflakeS3RestClient.get_file_headerc                 C  sH   t t | jji}| jr"|t t | jjt t | jj	t t
 | jji |S )zbConstruct metadata for a file to be uploaded.

        Returns: File metadata in a dict.

        )r   r   r'   sha256_digestr   rV   r   r   r   r   r   r   )rG   s3_metadatar$   r$   r%   _prepare_file_metadata  s   z,SnowflakeS3RestClient._prepare_file_metadatac                 C  s   d}t | jj| jjd }| |}| jd| d|  }|  }d}d| j	|< | j
|d||ttit|d}|jdkrQt|jd	 j| _d g| j | _d S |  d S )
N))uploadsr   r   ?Initiater   POST)r   r   r   r   rm   rb   r      )r   rB   r   r'   dst_file_namer   rl   rE   r   r   r   r   r   dictr   r   r   contentr   r?   num_of_chunksr@   r   )rG   rb   r   query_stringr   r   r   r   r$   r$   r%   _initiate_multipart_upload  s&   


z0SnowflakeS3RestClient._initiate_multipart_uploadchunk_idchunkc           
   	   C  s   t | jj| jjd }| jd|  }| jdkr2|  }| j	|d|||t
tidd}|  d S dt|d fd| jff}| |}| d| }	| j	|	d||dt|d	}|jd
kre|jd | j|< |  d S )Nr   r   PUTT)r   r   r   r   r   rm   r   
partNumberuploadIdr   )r   r   r   r   r   rb   r   ETag)r   rB   r   r'   r   r   rE   r   r   r   r   r   r   r   r?   rl   r   r   rm   r@   )
rG   r   r  r   r   r   r   rb   r   	chunk_urlr$   r$   r%   _upload_chunk  s<   
	

z#SnowflakeS3RestClient._upload_chunkc                 C  s   d| j ff}t| jj| jjd }| |}| jd| d|  }t	
d td}t| jD ],\}}td}td}	||	_||	 td}
t|d	 |
_||
 || q2d
}d| j|< | j|d|t|t|d}|  d S )Nr  r   r   z$Initiating multipart upload completeCompleteMultipartUploadPartr  
PartNumberr   Completer   r   )r   r   r   r   rb   )r?   r   rB   r   r'   r   r   rl   rE   rM   rN   r   Element	enumerater@   r   r   r   r   r   tostringr   r   )rG   rb   r   r   r   rootidxetag_strpartetagpart_numberr   r   r$   r$   r%   _complete_multipart_upload  s2   








z0SnowflakeS3RestClient._complete_multipart_uploadc                 C  s   | j d u rd S d| j ff}t| jj| jjd }| |}| jd| d|  }d}d| j	|< | j
|d|t|d}|  d S )Nr  r   r   Abortr   DELETEr   r   r   rb   )r?   r   rB   r   r'   r   r   rl   rE   r   r   r   r   )rG   rb   r   r   r   r   r   r$   r$   r%   _abort_multipart_upload!  s   


z-SnowflakeS3RestClient._abort_multipart_uploadc                 C  s  t d|  t| jj| jjd }| jd|  }| j	dkrB| j
|d|dd}|jdkr<| d|j tj| j_|  d S | j}|| j	d k r\||  d	|d | d  }n||  d	}| j
|d|d
d| id}|jdv r}| ||j |  d S )NzDownloading chunk r   r   GETT)r   r   r   r   r   r   -Rangezbytes=)r   r   r   rm   )r      )rM   rN   r   rB   r   r'   src_file_namer   rE   r   r   r   write_downloaded_chunkr   r   
DOWNLOADEDr   r   r+   )rG   r   r   r   r   r+   _ranger$   r$   r%   download_chunk3  s6   


 
z$SnowflakeS3RestClient.download_chunkr   c                 C  s   d}|  |}d| d| }d}d| j|< | j|d|t|d}|jdkrXt|j}|jd |j	d	d
  }| d}	|
|	}
|
d u rIdn|
jdk}td|  |S dS )N))
accelerater   r:   z.s3.amazonaws.com/?r#  r   r  r  r   }r   StatusFEnabledzuse_accelerate_endpoint: )rl   r   r   r   r   r   r   r   tagindexrK   rM   rN   )rG   r   rb   r   r   r   r   config	namespace	statusTagfoundr-   r$   r$   r%   rO   S  s$   




z3SnowflakeS3RestClient._get_bucket_accelerate_config)NFF)r'   r   r(   r   r)   r*   r+   r,   r-   r.   r/   r0   r1   r0   r2   r3   )N)r-   r.   r2   r0   )rP   rQ   rR   r   r2   rQ   )rR   rQ   r2   rQ   )rb   rc   r2   r   )rm   rn   r2   ro   )Nr   )r   r   r   r   rb   r   r   r   r   r   r2   ro   )r>   r   r   r   r   r   r   r   r   rQ   r2   ro   )r   r   r2   r0   )r2   r   )NNNNFF)r   r   r   r   r   r   rb   r   r   r   rm   r   r   r   r   r0   r   r0   r2   r   )r   r   r2   r   )r2   r*   )r2   r3   )r   r,   r  rQ   r2   r3   )r   r,   r2   r3   )r   r   r2   r0   )r    r!   r"   r=   rF   staticmethodrZ   r]   r`   ra   rl   r   r   r   r   rA   r   r   r   r   r  r  r  r"  rO   __classcell__r$   r$   rH   r%   r&   3   sV    3,	$

[
,


$

 r&   )8
__future__r   r[   rexml.etree.ElementTreeetreeElementTreer   r   r   ior   loggingr   operatorr   typingr   r	   r
   cryptography.hazmat.primitivesr   r   compatr   r   	constantsr   r   r   r   encryption_utilr   storage_clientr   r   vendoredr   file_transfer_agentr   r   r    rM   r   r   r   r   r   ERRORNO_WSAECONNABORTEDr   ADDRESSING_STYLEr   compiler   r   r&   r$   r$   r$   r%   <module>   s<    
