o QDib@spddlmZddlmZddlmZmZmZmZm Z m Z m Z ddl Z ddl mZddlmZmZmZmZddlmZdd lmZdd lmZdd lmZdd lmZdd lmZddlmZddl m!Z!ddl"m#Z#erddlm$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*m+Z+ddl,m-Z-m.Z.ddl/m0Z0ddl1m2Z2ddlm3Z3e j4dddGdddee#dZ5d%d#d$Z6dS)&) annotations)Pool) TYPE_CHECKINGAnyClassVarDictListOptionalUnionN)Self)errorsfieldshelperssettings) Checklist)FrictionlessException)Metadata)platform)ReportResource)system) Transformer)Factory)types)Dataset)Detector)ControlDialect) IOnProgressIOnRow)Pipeline) TableResource) PublishResultTF)kw_onlyreprcs:eZdZUdZejdddZded< dZded< ejdd d Z d ed < dZ d ed < dZ ded< dZ d ed< dZ d ed< dZd ed< dZd ed< ejedZded< ejedZded< ejedZded< ejedZded< dZd ed< dZd ed< dZd ed< ejedZded< dZd ed!< ejdd"d Zd#ed$< ejdd%d Zd&ed'< fd(d)Zedd+d,Zejdd.d,Zedd/d0Z edd1d2Z!dd6d7Z"dd:d;Z#ddd?Z%ddAdBZ&ddDdEZ'ddHdIZ(ddJdKZ)dLdMZ*dNdOZ+ddPddSdTZ, dddUddYdZZ-d d[gfdd]d^Z.d_d`Z/ddaddcddZ0e1 dddPddfdgZ2dddddhddpdqZ3dddddddrdd{d|Z4dddZ5 dddde6j7ddddZ8dfdd Z9ddddZ:dZ;eddgde6j?dde6j@dddiddiddiddiddddiddiddidddddddiddiddidddddddiddiddiddiddiddddddidddiddiddidddidddZAe1dddZBe1dddZCe1dfdd ZDe1dfdd ZEe1dfdd ZFfddZGZHS)PackageuPackage representation This class is one of the cornerstones of of Frictionless framework. It manages underlaying resource and provides an ability to describe a package. ```python package = Package(resources=[Resource(path="data/table.csv")]) package.get_resoure('table').read_rows() == [ {'id': 1, 'name': 'english'}, {'id': 2, 'name': '中国人'}, NF)defaultr& Optional[Any]sourceOptional[Control]controlbasepath)r)alias Optional[str] _basepathnamezClassVar[Union[str, None]]typetitle descriptionhomepageprofile)factoryzList[Dict[str, Any]]licensessources contributors List[str]keywordsimageversioncreatedzList[Resource] resourceszOptional[Dataset]datasetdialectzOptional[Dialect]_dialectdetectorzOptional[Detector] _detectorcs<|jD]}||_|jr|j|_|jr|j|_qtdSN)rApackagerDrCrFrEsuper__attrs_post_init__selfresource __class__U/var/www/Datamplify/venv/lib/python3.10/site-packages/frictionless/package/package.pyrJs zPackage.__attrs_post_init__returncCs|jr|jS|jr |jjSdS)zq A basepath of the package The normpath of the resource is joined `basepath` and `/path` N)r1rBr.rLrPrPrQr.s zPackage.basepathvaluecCs ||_dSrG)r1)rLrTrPrPrQr. cCdd|jDS)Return names of resourcescSg|] }|jdur|jqSrGr2.0rMrPrPrQ z*Package.resource_names..rArSrPrPrQresource_nameszPackage.resource_namescCrV)rWcSrXrG)pathrZrPrPrQr\r]z*Package.resource_paths..r^rSrPrPrQresource_pathsr`zPackage.resource_pathsrMUnion[Resource, str]rcCs0t|tr tj||jd}|j|||_|S)zAdd new resource to the package)r.) isinstancestrrfrom_descriptorr.rAappendrHrKrPrPrQ add_resources  zPackage.add_resourcereboolcCs |jD] }|j|kr dSqdS)zCheck if a resource is presentTF)rAr2rLr2rMrPrPrQ has_resources  zPackage.has_resourcecCs.|jD]}|j|krt|tjjrdSqdS)z$Check if a table resource is presentTF)rAr2rdrfrictionless_resourcesr$rjrPrPrQhas_table_resources  zPackage.has_table_resourcecCs:|jD] }|j|kr|Sqtjd|dd}t|)zGet resource by name resource "z" does not existnote)rAr2r PackageErrorrrLr2rMerrorrPrPrQ get_resources  zPackage.get_resourcer$cCs8||}t|tjjr|Stjd|dd}t|)z/Get table resource by name (raise if not table)rnz" is not tabularro)rtrdrrlr$r rqrrrrPrPrQget_table_resources zPackage.get_table_resourceOptional[Resource]cCsP|jsJ||jr!||j}|j|}||j|<||_|S||dS)zSet resource by nameN)r2rkrtrAindexrHrh)rLrM prev_resourcerwrPrPrQ set_resources     zPackage.set_resource descriptortypes.IDescriptorcCsF||}|j|}|}||t|}||_||j|<|S)zUpdate resource)rtrArw to_descriptorupdaterrfrH)rLr2rzrxresource_indexresource_descriptor new_resourcerPrPrQupdate_resources     zPackage.update_resourcecCs||}|j||S)zRemove resource by name)rtrAremoverjrPrPrQremove_resources  zPackage.remove_resourcecCs g|_dS)zRemove all the resourcesNr^rSrPrPrQclear_resources rUzPackage.clear_resourcescCsrt|jtt|jkr5g}t|jD]#\}}|j}||d}|dkr/d||f|j|_||qdSdS)Nrz%s%s)lenr_set enumeraterAr2countrg)rL seen_namesrwrMr2rrPrPrQdeduplicate_resouress zPackage.deduplicate_resouresstatsrNonecCs|jD]}|j|dqdS)zgInfer metadata Parameters: stats: stream files completely and infer stats rN)rAinfer)rLrrMrPrPrQrs z Package.infer)r-targetrr%cCsBtj||dd}|std|d||}|std|S)a'Publish package to any supported data portal Parameters: target (string): url e.g. "https://github.com/frictionlessdata/repository-demo" of target[CKAN/Github...] control (dict): Github control Returns: Any: Response from the target T)r- packagifyzNot supported target: z or controlzNot supported action)rcreate_adapterr write_packageto_copy)rLrr-adapterrarPrPrQpublish(s zPackage.publishraspeccs>g}|jD]}i||fdd|Dq|S)zFlatten the package Parameters spec (str[]): flatten specification Returns: any[]: flatten package csg|]}|qSrP)get)r[propcontextrPrQr\Ksz#Package.flatten..)rAr}r|rg)rLrresultrMrPrrQflatten>s  zPackage.flattencCs|jD]}|qdS)zDereference underlaying metadata If some of underlaying metadata is provided as a string it will replace it by the metadata object N)rA dereferencerKrPrPrQrPs  zPackage.dereferencedetailedrcCs4i}|jD]}t|tjjr|j|d||j<q|S)aAnalyze the resources of the package This feature is currently experimental, and its API may change without warning. Parameters: detailed? (bool): detailed analysis Returns: dict: dict of resource analysis r)rArdrrlr$analyzer2)rLranalysisrMrPrPrQr[s  zPackage.analyzeoptionscKs*tj|fd|d|}t|tsJ|S)aDescribe the given source as a package Parameters: source (any): data source stats? (bool): if `True` infer resource's stats **options (dict): Package constructor options Returns: Package: data package rH)r3r)rdescriberdr()clsr+rrmetadatarPrPrQrpszPackage.describe)r2filterprocess limit_rowsrOptional[types.IFilterFunction]r Optional[types.IProcessFunction]r Optional[int]types.ITabularDatac CsTi}|dur |jn||g}|D]}t|tjjr'|j|||d}||q|S)aExtract rows Parameters: filter: row filter function process: row processor function limit_rows: limit amount of rows to this number Returns: extracted rows indexed by resource name N)rrr)rArtrdrrlr$extractr}) rLr2rrrdatarAresitemrPrPrQrs zPackage.extract)r2faston_row on_progress use_fallbackqsv_path database_urlrrOptional[IOnRow]rOptional[IOnProgress]rrc CsVg}|dur |jn||g} | D]} t| tjjr(|| j||||||dq|S)N)rrrrrr)rArtrdrrlr$extendrw) rLrr2rrrrrnamesrArMrPrPrQrws   z Package.indexrLpipeliner#cCst}|||S)zTransform package Parameters: source (any): data source steps (Step[]): transform steps **options (dict): Package constructor options Returns: Package: the transform result )rtransform_package)rLr transformerrPrPrQ transforms zPackage.transform)r2parallelr limit_errors checklistOptional[Checklist]rrintc Csxt}g}|dur |jn||g}tdd|D} |p t}z|jddWntyE} ztj |j | dWYd} ~ Sd} ~ ww|rJ| r]|D]} | j |||d} | | qLnWtL} g}|D],} i}i|d<| |dd <| j|dd <i|d <||d d <||d d <| |qe| t|}|D] }| t|qWdn1swYtj|j |dS)a7Validate package Parameters: checklist? (checklist): a Checklist object parallel? (bool): run in parallel if possible. Parallel execution is not possible if foreign keys are used in a resource schema. Returns: Report: validation report Ncss|] }|jo |jjVqdSrG)schema foreign_keys)r[rrPrPrQ s z#Package.validate..T)validate)timer )rrrrMrzr.rrr)rreports)rTimerrArtanyrr|rrfrom_validationr to_errorsrrgrr.map_validate_parallelrffrom_validation_reports)rLrr2rrrtimerrrAwith_foreign_keys exceptionrMreportpool options_poolrreport_descriptorsreport_descriptorrPrPrQrsT        zPackage.validater c s*tjddd|jD|j|jd|S)zCreate a copy of the packagecSsg|]}|qSrP)rrZrPrPrQr\sz#Package.to_copy..)rAr.rBNrP)rIrrAr.rB)rLrrNrPrQrszPackage.to_copyc Cs`tjj}||}|r.z t||W|Sty-}z tt j t |d|d}~ww|S)aFGenerate ERD(Entity Relationship Diagram) from package resources and exports it as .dot file Based on: - https://github.com/frictionlessdata/frictionless-py/issues/1118 Parameters: path (str): target path Returns: path(str): location of the .dot file roN) rfrictionless_formatserd ErdMapperrr write_file Exceptionrr rqre)rLramappertextexcrPrPrQ to_er_diagram$s zPackage.to_er_diagramrHobjectstring)r3patternarray)r2rar4)r3 properties)r3items)r4raemail)r4rar organisationrole)r2r3r4r5r6r7r9r:r;r=r>r?r@rA)r3requiredrcCs t|SrG)rselect_package_class)rr3rPrPrQmetadata_select_classyrUzPackage.metadata_select_classcCs|dkrtSdS)NrAr)rr2rPrPrQmetadata_select_property_class}sz&Package.metadata_select_property_classcs\t||dd|dd}t|tr(|r*t|dtr,|d|d<dSdSdSdS)Nz $frictionlessprofilesrr7)rImetadata_transformpoprdlistre)rrzrrNrPrQrs   zPackage.metadata_transformc#stt|}|r|EdHdStjsLdg}|D]1}||}t|tr(|n|g}|D]}|rJt|trJt |sJt j d|ddVdSq-qg}|dD]} t| t rdd| vrd| | dqRt|tt|krxd} t j | dV|d} | rtjdd } | | \} } | rd } t j | dV|d gD]}|d s|dsd |} t j | dVqdD]0}||gD]'}|drtjddd} | |d\} } | rd|d} t j | dVqq|d}|r|dvr|dkrd}tj|||jdEdH|dg}|dkr+|D]} | dddkr)d} t j | dVqdD]}||vrDd|d|d} t j | dVq-dS)Nr7zpath "z " is not saferorAr2z%names of the resources are not uniquer@rYz*property "created" is not valid "datetime"r9raz#license requires "path" or "name": )r;r:r)r2formatz property "z[].email" is not valid "email")z data-packagetabular-data-packagezfiscal-data-packagezBhttps://specs.frictionlessdata.io/schemas/fiscal-data-package.json)r7 error_classrztabular-data-resourcezWprofile "tabular-data-package" requires all the resources to be "tabular-data-resource") missingValuesr"z$" should be set as "resource.schema.)rrImetadata_validatertrustedrrdrer is_safe_pathr rqdictrgrrr DatetimeField read_cell StringFieldrmetadata_Error)rrzmetadata_errorskeyskeyrTrrr_rMrpr@field_r2r7rArNrPrQrs            zPackage.metadata_validatec stjd|dd|S)NT)rz with_basepathrP)rImetadata_import)rrzrrNrPrQrs zPackage.metadata_importcst}|SrG)rImetadata_export)rLrzrNrPrQrs zPackage.metadata_export)rRr0)rTr0)rRr<)rMrcrRr)r2rerRri)r2rerRr)r2rerRr$)rMrrRrv)r2rerzr{rRr)rrirRrrG)rrr-r,rRr%)rr<)rri)r+r*rrirr) r2r0rrrrrrrRr)rrer2r0rrirrrrrrirr0rRr<)rLr(rr#) rLr(rrr2r0rrirrrr)rrrRr )rar0rRre)r3r0)r2re)rzr{)rzr{rr)I__name__ __module__ __qualname____doc__attrsrr+__annotations__r-r1r2r3r4r5r6r7rr9r:r;r=r>r?r@rArBrDrFrJpropertyr.setterr_rbrhrkrmrtruryrrrrrrrrr classmethodrrrwrrDEFAULT_LIMIT_ERRORSrrr metadata_typer rqr NAME_PATTERN TYPE_PATTERNmetadata_profilerrrrrr __classcell__rPrPrNrQr(s4                            ! H     ;   Nr() metaclassrr{rRcCs8|d}|d}tjdi|}|jdi|}|S)NrMrrP)rrfrr|)rresource_optionsvalidate_optionsrMrrPrPrQrs r)rr{rRr{)7 __future__rmultiprocessingrtypingrrrrrr r r typing_extensionsr r rrrrrrrrrrrrrMrrrrr8rrcatalogrrErrCrr indexerr!r"rr#rAr$r%definer(rrPrPrPrQs@ $                 V