o QDiz@sRddlmZddlZddlZddlmZmZmZmZm Z m Z m Z m Z ddl Z ddlmZddlmZmZmZmZddlmZddlmZdd lmZmZdd lmZdd lmZdd l m Z dd l!m"Z"ddl#m$Z$ddl%m%Z%ddl&m'Z'ddl(m)Z)erddlm*Z*ddl+m,Z,ddl-m.Z.ddl%m/Z/e j0dddGdddee'dZ1dS)) annotationsN) TYPE_CHECKINGAnyClassVarDictListOptionalUnioncast)Self)errorsfieldshelperssettings) Checklist)Detector)ControlDialect)FrictionlessException)Metadata)platform)Report)Schema)system)Factory) ResourceStats)types)Error)Package)LoaderTF)kw_onlyreprc seZdZUdZejdddZded< dZded< dZ d ed < ejd d d Z ded< ded< dZ ded< dZ ded< dZ ded< dZded< ejedZded< ejedZded< dZded< dZded< dZded< dZded< ejd dd Zded< dZded < dZded!< ejedZd"ed#< dZded$< dZded%< dZded&< dZd'ed(< dZd'ed)< dZd'ed*< eje d+d,Z!d-ed.< eje"d/d,Z#d0ed1< ejdd2d Z$ded3< eje%dZ&d4ed5< dZ'd6ed7< ejdd8Z(d9ed:< dZ)d;ed<< fd=d>Z*d?d@Z+dAdBZ,e-ddDdEZ.e-ddFdGZ/e-ddHdIZ0e-ddKdLZ1e-ddMdNZ2e-ddOdPZ3e-ddQdRZ4e-ddTdUZ5e5j6ddWdUZ5e-ddYdZZ7e7j6dd[dZZ7e-dd\d]Z8e8j6dd^d]Z8e-dd`daZ9e-ddcddZ:e-ddfdgZ;e-ddhdiZe?ddodfdudvZ@ddwddydzZAddwdd{d|ZBddwdd}d~ZCdddddZDddZEe? ddddddddZFdddddZ dddddeGjHddddZIdʇfdd ZJdZKeLjMZNdd gid deGjOdddeGjPddddidddidddidddidddddiddiddiddddddddiddiddiddddddiddddgidddidddid ddid!ddid#dddidd$ddid%ddiddiddiddiddidddgidddgiddZQe?dddZRe?dddZSe?d͇fdd ZTe?d͇fdd ZUe?d·fdd ZVfddZWZXS)ResourceuResource representation. This class is one of the cornerstones of of Frictionless framework. It loads a data source, and allows you to stream its parsed contents. At the same time, it's a metadata class data description. ```python with Resource("data/table.csv") as resource: resource.header == ["id", "name"] resource.read_rows() == [ {'id': 1, 'name': 'english'}, {'id': 2, 'name': '中国人'}, ] ``` NF)defaultr" Optional[Any]sourcezOptional[Control]controlbool packagifyname)r%alias Optional[str]_namez ClassVar[str]typetitle descriptionhomepageprofile)factoryzList[Dict[str, Any]]licensessourcespathdataschemeformatdatatype _datatype mediatype compression List[str] extrapaths innerpathencodinghash Optional[int]bytesrrowsdialect)r5r-Union[Dialect, str]_dialectschemaUnion[Schema, str]_schemabasepath _basepathrdetectorzOptional[Package]package)initrstatszClassVar[bool]tabularcs|jpd|_|jp d|_t|_tt|dd}t|t r ||_d|_ d|_ t ||d|d|d|d|d|d|d tdS) Nr+r<r,r:r;r?r>rHrS)r/r,r=r<rrSgetattrr0 isinstancestr_Resource__loader_Resource__bufferrdetect_resource add_definedsuper__attrs_post_init__)selfr< __class__W/var/www/Datamplify/venv/lib/python3.10/site-packages/frictionless/resource/resource.pyr]s"           zResource.__attrs_post_init__cCs|jr||SN)closedopenr^rararb __enter__szResource.__enter__cCs |dSrc)close)r^r0value tracebackrararb__exit__s zResource.__exit__returncCs*g}|jdur ||j||j|S)zAll paths of the resourceN)r8appendextendrA)r^pathsrararbro s    zResource.pathscCs*g}|jD] }|tj||jdq|S)z Normalized paths of the resourcerN)rormrnormalize_pathrN)r^ normpathsr8rararbrrs zResource.normpathscCs|jr tj|j|jdSdS)z3Normalized path of the resource or raise if not setrpN)r8rrqrNrfrararbnormpathszResource.normpathrWcCsH|jdurdS|jr|jdS|jr|jd|jS|jr"|jSdS)zStringified resource locationNzz (multipart)z -> r+)r9rAr8rBrfrararbplace#s  zResource.placecCs |jduS)z"Whether resource is not path basedN)r9rfrararbmemory0s zResource.memorycCst|jp |jp dS)zWhether resource is remoter+)ris_remote_pathrNr8rfrararbremote5szResource.remotecCs|j ot|jS)zWhether resource is multipart)rur)rArfrararb multipart:szResource.multipartrcC&t|jtrtj|j|jd|_|jSNrp)rVrJrWrfrom_descriptorrNrfrararbrH? zResource.dialectricC ||_dSrc)rJr^rirararbrHE rcCryrz)rVrMrWrr{rNrfrararbrKIr|zResource.schemacCr}rc)rMr~rararbrKOrcCs|jr|jS|jr |jjSdS)zr A basepath of the resource The normpath of the resource is joined `basepath` and `/path` N)rOrQrNrfrararbrNSs zResource.basepathcCr}rc)rOr~rararbrN^r types.IBuffercCs|jdur td|jS)zFile's bytes used as a sample These buffer bytes are used to infer characteristics of the source file (e.g. encoding, ...). N"resource is not open or non binary)rYrrfrararbbufferds zResource.buffertypes.IByteStreamcC2|jrtd|jst||_|j|jjS)zaByte stream in form of a generator Yields: gen?: byte stream r)rdrrXr create_loaderre byte_streamrfrararbro   zResource.byte_streamtypes.ITextStreamcCr)zaText stream in form of a generator Yields: gen?: text stream z#resource is not open or non textual)rdrrXrrre text_streamrfrararbr}rzResource.text_streamcCs |jduS)zRWhether the table is closed Returns: bool: if closed N)rXrfrararbrds zResource.closedNonecCs|jr |jd|_dSdS)z+Close the resource as "filelike.close" doesN)rXrhrfrararbrhs  zResource.closecCsH|zt||_|j|jj|_W|Sty#|w)z#Open the resource as "io.open" does)rhrrrXrerrY Exceptionrfrararbres    z Resource.openr< descriptorUnion[types.IDescriptor, str]optionsrr c s@|rtj|d}|j|fi|}||_|Stj|fi|S)Nr)rselect_resource_classr{r<r\)clsrr<rr^r_rarbr{s  zResource.from_descriptorsizercCs|jrdSt|,|s)d} tt|j}||7}|snq|WdS|j|WdS1s9wYdS)zURead bytes into memory Returns: any[][]: resource bytes TN)rur ensure_openr rFrread1)r^rrchunkrararb read_bytess  $zResource.read_bytescCsD|jrdSt||j|WdS1swYdS)zORead text into memory Returns: str: resource text r+N)rurrrread)r^rrararb read_texts   $zResource.read_textcCsX|jdur|jSt||j|d}t|}|WdS1s%wYdS)zORead data into memory Returns: any: resource data Nr)r9rrrjsonloads)r^rtextr9rararb read_datas    $zResource.read_datarScCs|js d}ttj|d|'|s WddSt|jd|jj|_ |jj |_ WddS1s9wYdS)zfInfer metadata Parameters: stats: stream file completely and infer stats z0Resource.infer cannot be used on a open resourcenoteNsha256:) rdrr ResourceErrorr pass_throughrrSsha256rDrF)r^rSrrararbinfers  "zResource.infercCs$d|j_d|j_d|j_d|j_dS)zDereference underlaying metadata If some of underlaying metadata is provided as a string it will replace it by the metadata object N)rHmetadata_descriptor_pathmetadata_descriptor_initialrKrfrararb dereferences zResource.dereference)r,r0rSrc Kstjj}tjj}t|f|pd|dkd|}|dvr5t||r5|} | j|d|dur3| |S| S|j|d|dkrV||gd} | j|d|durT| |S| S|dkr]|j S|d krd|j S|S) aDescribe the given source as a resource Parameters: source: data source name: resoucrce name type: data type: "package", "resource", "dialect", or "schema" stats: if `True` infer resource's stats **options: Resource constructor options Returns: Metadata: metadata describing this data source r+rQ)r,r*)rQNrN) resourcesrHrK) r frictionlessr frictionless_resourcesPackageResourcer$rV read_metadatar get_resourcerHrK) rr'r,r0rSrr rresourcerQrararbdescribe s8      zResource.describe)r,List[Resource]cCs|gS)zmList dataset resources Parameters: name: limit to one resource (if applicable) ra)r^r,rararblistKsz Resource.list)r,on_rowparallel limit_rows limit_errors checklistOptional[Checklist]r!Optional[types.ICallbackFunction]rrrintrc Csd}t}g} g} g} |pt}||} z|jddWnty;} ztj||j| dWYd} ~ Sd} ~ ww|j rRt |j \}}|dvrRd}| ||j r}z|Wn!ty|} z|tj||j| dWYd} ~ Sd} ~ ww|t| D]\}}|D]}|jdkr| |=||r| |qqt|tjjs|j dus|jdurt|jnd }|j} |d 7}zt|j}Wn ty} z | | jWYd} ~ qd} ~ wtyYnSw| D]}| |D] }||r| |qq|r |||r ||kr d |}| |d}n|r=t!| |kr=| d|} d |}| |d}nq|sZ| D]}|"D]}||rV| |qIqCWdn 1sewYtj||j| | | d S)aValidate resource Parameters: checklist: a Checklist object name: limit validation to one resource (if applicable) on_row: callbacke for every row paraller: allow parallel validation (multiprocessing) limit_rows: limit amount of rows to this number limit_errors: limit amount of errors to this number Returns: Report: validation report FT)validate)timer Nmd5rz1hash is ignored; supported algorithms: md5/sha256z check-errorrrzreached row limit: zreached error limit: )rlabelsr warnings)#rTimerrconnect to_descriptorrrfrom_validation_taskr to_errorsrDparse_resource_hash_v1rmrdrerh enumeratevalidate_startr0matchrVrr TableResourcerFrrrnext row_streamerror StopIteration validate_rowlen validate_end)r^rr,rrrrpartialtimerrr rchecks exception algorithm_warningindexcheckr row_countrowrararbrVs                      (   A zResource.validatec s$tjd|j|j|j|jd|S)zCreate a copy from the resource)r9rNrPrQNra)r\to_copyr9rNrPrQ)r^rr_rarbrszResource.to_copyrobjectstring)r0patternarray)r,r8r1)r0 properties)r0items)r1r8email)r0iteminteger)rDrFrrGrHrK)r0requiredrcCs t|Src)rr)rr0rararbmetadata_select_classrzResource.metadata_select_classcCs|dkrtS|dkr tSdS)NrHrK)rr)rr,rararbmetadata_select_property_classs z'Resource.metadata_select_property_classtypes.IDescriptorcst||dd}|d}|d}|s$|dur$|r$|d||d}|r>t|tr>|d|d<|dd|d<|dd}t|trX|rXt|dtrX|d|d<|d d}|rl|d i||d d <|d d}|d d}|r|r|d d} | r| ||<d } | d7} t | t |d d} | rt | \} } | dvr|d |d i| |d | <|d d}|rt|t r|dd} |dd} | rd| |d <n| r| |d <dD] }||}|r|||<q|d}|dkr |dd} | d7} t | t |dd}|r|dkrd|d<|dd}|rB|di|d|d} | d7} t | t dSdS)Nurlr8r9rrrAprofilesr4rFrShashingrDzBResource "stats.hash" is deprecated in favor of "stats.sha256/md5"z.(it will be removed in the next major version)rrrr)rFrrGr?noz>Resource "compression=no" is deprecated in favor not set valuetabular-data-resourcetabler0layoutrHz5Resource "layout" is deprecated in favor of "dialect")r\metadata_transformpopget setdefaultrVrrWrwarn UserWarningrrdictupdate)rrrr8r9rrFrrSrDralgorrr,rir?r4rr_rarbr&s                                zResource.metadata_transformc#s$tt|}|r|EdHdStjsMgd}|D]1}||}t|tr)|n|g}|D]}|rKt|trKt |sKt j d|ddVdSq.q|d}|d} |durh| durhd} t j | dV|dury| duryd} t j | dV|d gD]}|ds|d sd |} t j | dVqd D]0} || gD]'}|d rt j d d d} | |d \} } | rd| d} t j | dVqq|d}|r|dvrtj|||jdEdH|d}|dkr|sd} t j | dVdD]} | |vrd| d| d} t j | dVqdS)N)r8rAr4rHrKzpath "z " is not saferr8r9z2one of the properties "path" or "data" is requiredz2properties "path" and "data" is mutually exclusiver6r,z#license requires "path" or "name": ) contributorsr7r)r,r;z property "z[].email" is not valid "email"r4)z data-resourcer)r4 error_classrKrz?profile "tabular-data-resource" requires "schema" to be present) missingValues"z" should be set as "schema.)rr\metadata_validatertrustedrrVrWr is_safe_pathr rr StringField read_cellrmetadata_Error)rrmetadata_errorskeyskeyrirrr8r9rr,fieldrr4rKr_rarbr|sr          zResource.metadata_validatec stjd|dd|S)NT)r with_basepathra)r\metadata_import)rrrr_rarbr s zResource.metadata_importc st}|d}ttttttf}|durt ||sg|d<t j dkrE|d}| dd}|rEg|d<|r>|d ||d|t j dkr|| dd}|r||d}|d}|d} |durld ||d <|durt||d <| dur|| |d<|S) Nr9v1r8rArSrrrFrrD)r\metadata_exportrrWr)rfloatrrrVr standardsrrmrn) r^rr9rr8rArSrrrFr_rarbr s4          zResource.metadata_export)rlr@)rlr.)rlrW)rlr))rlr)rirI)rlr)rirL)rir.)rlr)rlr)rlr)rlr)rrr<r.rrrlr )rrErlrF)rrErlrW)rrErlr)rSr)rlrrc) r'r&r,r.r0r.rSr)rrrlr)r,r.rlr)rrr,r.rrrr)rrErrrlr)rrrlr )r0r.)r,rW)rr)rrrr)Y__name__ __module__ __qualname____doc__attrsr r'__annotations__r(r*r/r1r2r3r4rr6r7r8r9r:r;r=r>r?rArBrCrDrFrrGrrJrrMrOrrPrQrSrTr]rgrkpropertyrorrrsrtrurwrxrHsetterrKrNrrrrdrhre classmethodr{rrrrrrrDEFAULT_LIMIT_ERRORSrr metadata_typer rr NAME_PATTERN TYPE_PATTERNmetadata_profilerrrrr r  __classcell__rarar_rbr$s                                          >      !"#$%&  2  UCr$) metaclass)2 __future__rrrtypingrrrrrrr r rtyping_extensionsr r+r rrrrrrPrrHrrrrmetadatarrreportrrKrrr5rrSrrrrrQr r!definer$rarararbs2 (