o
    Cil                    @   s  d dl Z d dlmZ d dlmZ d dlmZ d dlmZ d dlm	Z	 d dlm
Z
 d dlmZ G dd	 d	e
Zd d
lmZ dd Zd dlmZ d dlmZ d dlmZmZ d dlmZ d dlZd dlmZ d dlmZmZ d dlmZm Z m!Z!m"Z" d dl#m$Z$ dZ%dZ&dZ'dZ(dZ)i dd dD Z*i dd dD Z+i dd dD Z,i d d d!D Z-i d"d d#D Z.d$d%d&d'd(d)d&d&d&d&d&d&d&d&d*Z/i d+d d,D d-d d.D Z0d/d0d0d1Z1i d2d d3D Z2i d4d d5D Z3i d6d d7D Z4i d8d d9D Z5d:d;d<d=d>d?d@Z6i dAd dBD dCd dDD dEd dFD Z7i dGd dHD Z8i dId dJD dKd dLD Z9i dMdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcddd_dedfdgd0dhdidjd_dkdldmdndodpdodqdrZ:i dsd dtD Z;dud/dpd(dvdwdxdydzd{d|
Z<i d}d d~D Z=G dd dZ>G dd dZ?G dd dZ@G dd dZAG dd dZBG dd dZCG dd dZDG dd dZEG dd dZFG dd dZGG dd dZHG dd dZIG dd dZJd dlKZKG dd dZLd dl mMZM d dlNmOZO d dlPmQZQ d dlRmSZSmTZTmUZUmVZV G dd dZWG dd dZXG dd dZYd dlZZ[de[j\de]de[j\fddZ^d dl_Z_d dl`maZa d dlbZbde[j\de[j\fddZcde]fddZdG dd dZedS )    N)defaultdict)Response)status)AuthenticationFailed)ValidationError)APIExceptionc                   @   s   e Zd ZejZdZdZdS )InvalidCredentialsExceptionInvalid Credentialsinvalid_credentialsN)__name__
__module____qualname__r   HTTP_401_UNAUTHORIZEDstatus_codedefault_detaildefault_code r   r   ,/var/www/Datamplify/integration_injection.pyr      s    r   )exception_handlerc                 C   s0   t | |}|d urd|jv rd|jd i|_|S )Ndetailmessage)r   data)exccontextresponser   r   r   custom_exception_handler   s
   

r   )uuid4)StringIO)DictList)Integrations)execute_batch)encrypt_jsondecrypt_json)convert_percentage_columnsconvert_numeric_columnsconvert_datetime_columnsmap_dtypes_to_clickhouse)Integration_Credentialsz*kdz9woqvtg.ap-south-1.aws.clickhouse.cloudi   defaultz9N0.fysGGTqBVc                 C      i | ]}|d diqS )typepager   .0er   r   r   
<dictcomp>4   
    r0   )EProfitAndLossDetailTransactionListTrialBalanceVendorExpensesAgedPayableDetailAgedReceivableDetailCustomerBalanceCustomerBalanceDetailCashFlowAgedReceivablesAccountListCustomerIncomeVendorBalanceGeneralLedgerCompanyInfobalance_sheetprofitandlossAccountbillCustomerEmployeeestimateInvoiceItemPayment	TaxAgencyvendorBudgetClasscompanycurrency
CreditMemocreditcardpaymentCustomerType
DepartmentDepositexchangeratejournalcodeJournalEntryPaymentMethodPreferencesPurchasePurchaseOrderRecurringTransactionRefundReceiptReimburseChargeSalesReceiptTaxCodeTaxRateTermTimeActivityTransfervendorcredit
attachablebillpaymentAgedPayablesr8   r9   r:   r;   r<   r=   r>   r?   r2   r3   r4   r5   r6   r7   c                 C   s   i | ]	}|d dddqS )r,   qnextRecordsUrl)r+   
page_paramcursor_paramr   r-   r   r   r   r0   B   s    
)*AnnouncementAppMenuItemAsset
AttachmentCampaignCaseGroupContactContract	DashboardDocumentDomainEmailCaptureEntitlementEventFileSearchActivityFolderrs   HolidayIdea
IndividualLeadLocationMacroNoteOpportunityOrderOrganizationPartnerPeriodProfile	PublisherRecordActionReportShipmentSiteSkillSolutionStampTaskUserUserAppMenuCustomizationc                 C      i | ]
}|d d dddqS )r,   limit  r+   rk   page_size_param	page_sizer   r-   r   r   r   r0   Q       
)	companiesproductsorderssubscriptionsinvoiceszinvoices/draftItemsc              
   C   s    i | ]}|d ddddddqS )r,   startAtisLast
maxResults   )r+   rk   rl   cursor_fieldr   r   r   r-   r   r   r   r0   [       
)announcementBannerapplicationroleauditing/recordissuesecurityschemesissuetypeschemeeventsissue/limit/reportzapplication-propertiesinstance/licensezlicense/approximateLicenseCountpermissions
plans/planpriorityschemerolescreenschemescreens/tabsscreenszconfiguration/timetrackingworkflowworkflowschemelabelprojectCategory	componentzproject/typeproject
serverInfousersstatuscategoryc                 C      i | ]
}|d ddddqS )r,   offsetr   d   r   r   r-   r   r   r   r0      r   )accountsencryptionsenvironmentsinvitesjobslicensesnotificationsprojectsrepositoriesrunsr   recordsissueSecuritySchemesvalueslimitsapplicationsr   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   c              
   C       i | ]}|d d dddddqS )r,   hasMorer   itemsr   )r+   rk   rl   r   
result_keyr   r   r-   r   r   r   r0      r   )formswebhooks
workspacesc                 C   s   i | ]}|d ddqS )Noner   )r+   r   r   r-   r   r   r   r0      s    
)users/mer   r   )r   r   r   c              
   C   r   )cursornextafterpageSizeresults)r+   rl   r   cursor_inner_fieldr   r   r   r-   r   r   r   r0      r   )Tzcrm/v3/objects/contactszcrm/v3/objects/companieszcrm/v3/objects/dealszcrm/v3/objects/ticketscrm/v3/objects/quoteszcrm/v3/ownerszcrm/v3/schemaszmarketing/v3/emails/zmarketing/v3/forms/zcms/v3/blogs/postszsettings/v3/userszcontacts/v1/listszcrm/v3/imports/zcrm/v3/objects/customzcrm/v3/objects/coursescrm/v3/schemas/quoteszcrm/v3/schemas/companiescrm/v3/objects/ownerszcrm/v3/objects/marketing_eventszcrm/v3/objects/userszcrm/v3/lists/crm/v3/objects/cartsr   r   zcrm/v3/schemas/courseszcrm/v3/schemas/contactszcrm/v3/schemas/dealszcrm/v3/objects/goalszcrm/v3/objects/leadszcrm/v3/objects/line_itemszcrm/v3/objects/orderszcrm/v3/objects/partner-clientszcrm/v3/objects/productszcrm/v3/objects/serviceszcrm/v3/pipelines/orderszcrm/v3/schemas/customzcrm/v3/schemas/cartszcrm/v3/schemas/invoiceszcrm/v3/schemas/listingszcrm/v3/schemas/commercepaymentszcrm/v3/schemas/ordersr   zcrm/v3/schemas/subscriptionszmarketing/v3/campaigns/z2crm/v4/associations/definitions/configurations/allr   zcrm/v3/objects/discountszcrm/v3/objects/feeszcrm/v3/objects/invoicesz crm/v3/objects/commerce_paymentszcrm/v3/objects/subscriptionszcrm/v3/objects/taxesz!crm/v3/limits/associations/labelsz#crm/v3/limits/calculated-propertiesz!crm/v3/limits/custom-object-typeszcrm/v3/limits/custom-propertieszcrm/v3/limits/pipelinesz'crm/v3/limits/associations/records/fromzcrm/v3/limits/recordszcrm/v3/lists/foldersz#crm/v3/objects/feedback_submissionszcrm/v3/objects/goal_targetszcrm/v3/objects/partner_clientszcrm-object-schemas/v3/schemaszcrm/v3/owners/z#marketing/v3/emails/statistics/listzmarketing/v4/email/single-sendzmarketing/v3/marketing-events/z(communication-preferences/v4/definitionszaccount-info/v3/detailsz,account-info/v3/api-usage/daily/private-appszcms/v3/blogs/authorszcontent/api/v2/blogszblogs/v3/topicszcms/v3/domains/zcms/v3/hubdb/tableszcms/v3/hubdb/tables/draftzcontent/api/v2/layoutszcms/v3/pages/site-pageszcms/v3/site-search/searchzcontent/api/v2/templateszcms/v3/url-redirects/z#account-info/v3/activity/audit-logszsettings/v3/currencies/codesc                 C   "   i | ]}|d d ddddddqS )r,   infomore_recordsper_pager   Fr+   rk   r   rl   r   r   organizationsr   r-   r   r   r   r0         
)leadsdealscontactsc                 C   r   r,   r   r   r   NTr   r   r-   r   r   r   r0     r   )r   r   r   salesorderspackagesr   purchaseorderscreditnotes
pricebooksbillsr   	locationssalesreturnscompositeitems
itemgroups
categoriesc                 C   r   r   r   r-   r   r   r   r0   %  r   )r   	estimatesr   r   recurringinvoicesr   customerpaymentsexpensesrecurringexpensesretainerinvoicesr   r   recurringbillsvendorcreditsvendorpaymentsbankaccountsbanktransactionschartofaccountsjournalsfixedassetsbasecurrencyadjustmentr   r   r   r   base_currency_adjustmentsvendor_creditsrecurring_billsrecurring_expensesrecurring_invoicescomposite_items)r	  r  r  r   r   r   c              	   C   s   i | ]}|d d ddddqS )r   namer   r   )r+   rl   r   r   r   r   r-   r   r   r   r0   :  s    
)zqueries/custom-fields-detailedzqueries/custom-fieldsz%queries/scoped-custom-fields-detailedzqueries/scoped-custom-fieldszqueries/software-patch-installszqueries/antivirus-statuszqueries/antivirus-threatszqueries/computer-systemszqueries/device-healthzqueries/backup/usagezqueries/diskszqueries/logged-on-userszqueries/network-interfaceszqueries/operating-systemszqueries/os-patcheszqueries/software-patcheszqueries/policy-overrideszqueries/processorszqueries/raid-controllerszqueries/raid-driveszqueries/softwarezqueries/volumeszqueries/windows-servicesc              
   C   s    i | ]}|d ddddddqS )r,   r   idr   N)r+   rk   r   r   order_byr   r   r-   r   r   r   r0   U  r   )deviceszdevices-detailedr   zorganizations-detailed
activitiesr   c                 C   r*   r+   r   r   r-   r   r   r   r0   e  r1   )r   alertsr   policieszautomation/scriptsznotification-channels/enabledgroupsrolesznotification-channelstaskszsoftware-productszdevices/searchzticketing/attributeszrelated-itemszknowledgebase/folderzorganization/documentszdocument-templateszchecklist/templateszorganization/checkliststagzknowledgebase/transferc                 C   r   r,   r   r   r   r   r-   r   r   r   r0   t  r   )wzcompany/companieszcompany/configurationszservice/ticketszcompany/contactszsales/stageszsales/opportunitieszsales/probabilitieszsystem/departmentszprocurement/catalogzcompany/countrieszmarketing/groups/infozfinance/agreementszfinance/agreements/typeszproject/projectszproject/ticketszproject/projectTypeszfinance/billingStatuseszexpense/paymentTypeszexpense/paymentTypes/infozsales/activitieszsales/activities/typeszcompany/addressFormatszsystem/allowedfiletypes/zsystem/allowedorigins/zsystem/apiMemberszsystem/audittrailzsystem/authAnvilszfinance/billingCycleszfinance/billingSetupszfinance/billingTermszservice/boardszservice/info/boardtypeszschedule/calendars/infozschedule/calendarszsystem/callbackszmarketing/campaignszmarketing/campaigns/statuseszmarketing/campaigns/typeszprocurement/categorieszprocurement/changeorderzsystem/certificationsztime/chargeCodeszexpense/classificationszservice/codeszsales/commissionszcompany/communicationTypeszfinance/companyFinance/zcompany/companies/infozcompany/marketDescriptionszcompany/noteTypeszcompany/ownershipTypeszcompany/companies/statuseszcompany/teamRoleszcompany/companyTypeAssociationszcompany/companies/typeszcompany/configurations/typeszcompany/configurations/statuseszsystem/connectwisehostedsetupszcompany/contacts/departmentszcompany/contacts/infozcompany/contacts/typesztime/workTypesztime/workRoleszsystem/workflowszprocurement/warehouseszsystem/userDefinedFieldszcompany/tracksztime/sheetsztime/entrieszsystem/myCompany/timeExpensezfinance/taxIntegrationszfinance/taxCodeszsystem/settings/zsystem/surveyszprocurement/subcategories/zservice/sourceszservice/SLAszsystem/skillszprocurement/shipmentmethodszservice/teamszservice/surveyszservice/serviceSignoffzservice/locationszservice/emailTemplateszsystem/securityroleszschedule/typeszschedule/statuseszschedule/detailszschedule/entrieszsales/salesTeamszsales/quotaszsales/roleszprocurement/rmaTagszprocurement/rmaStatuseszprocurement/rmaActionszsystem/quoteLinkSetupzprocurement/purchaseordersz!procurement/purchaseorderstatuseszproject/statuseszprocurement/typeszservice/prioritieszsystem/portalReportszcompany/portalConfigurationszschedule/portalcalendarszproject/phaseStatuseszsales/orderszsystem/memberszprocurement/manufacturerszsystem/kpiszfinance/invoiceszschedule/holidaysListszmarketing/groupszfinance/glAccountszsystem/experimentszexpense/entrieszsystem/emailConnectorszsystem/documentszfinance/currencieszsystem/myCompany/crmc                 C   r   )r,   page_nor   r   r   r   r-   r   r   r   r0     r   )Client	ToDoGroupr[   r   ServiceReleasePipelineRelease	QuotationProjects	OrderLine
SalesOrderOpportunitiesChatrI   rH   Ticketsro   SupplierCallLogrp   CRMNoter   ActionsClientContractc                 C   r*   r  r   r-   r   r   r   r0     s
    )-Webhook	UserRolesAssetTypeInfoAgentStatus	TimesheetTaxTagsTabsSlackDetailsTeamScheduleSalesMailbox
TicketTypeProductProductComponentPriorityOrganisationOnlineStatusRolesApplicationFeaturesEmailTemplateMailboxLookupLicenseInfo	LanguagesJiraDetailsInstancer~   
AssetGroupFeedbackToDOr{   CurrencyCategoryCAB
BudgetTypeTicketRulesAISuggestionAddressWorkdayWorkflowTimesheetEventSLAr   r  clientsr  r   r[   r   r   sitesr   servicesr!  releasepipelinesr"  releasesr#  quotesr$  ticketsr%  
orderlinesr&  r   r'  r(  chatsrI   rH   r   r)  assets	supplierscalllogattachmentsactionsreports	contracts)ro   r*  r+  rp   r,  r   r-  r.  c                 C   r   r  r   r-   r   r   r   r0     s    
)r   datasetscustom-reportsapplicant_tracking/applicationszapplicant_tracking/statuseszapplicant_tracking/locationszapplicant_tracking/hiring_leadszapplicant_tracking/jobsbenefit/member_benefitz%benefits/settings/deduction_types/allbenefitcoveragesemployeedependentscompany_informationemployees/directory
files/viewzmeta/time_off/policiesmeta/time_off/typesztime_off/whos_outz
meta/userszmeta/tablesztraining/typeztraining/categoryrj  memberszBenefit CoverageszEmployee Dependentsfieldsr   timeOffTypes)
rj  r   rk  rl  rm  rn  ro  rq  rr  rs  c                 C   r*   r  r   r-   r   r   r   r0   -  r1   )r   	customersr   price_rulesshipping_zonesscript_tagsc                   @   R   e Zd ZdZdZdZdd Zdd Zdd	 Zd
e	fddZ
dd
e	defddZdS )NinjaClientzhttps://api.ninjaone.com/v2zhttps://api.ninjaone.comr   c                 C   "   || _ | j d | _|| _|| _d S Naccess_tokentoken_metadatar  credentialsIntegration_idselfr  r  r  r   r   r   __init__8     
zNinjaClient.__init__c                 C      d| j  ddS NBearer application/jsonAuthorizationAcceptr  r  r   r   r   headers>     
zNinjaClient.headersc           	      C   s   dd l }d| j}t| jd tr|| jd n| jd }d|}ddi}d| jd | jd	 t|d
}tj	|||dd}|j
dkrd| }|| _tjj| jdjt| jd |d| _| jS td)Nr   z{}/oauth/tokenscopes Content-Type!application/x-www-form-urlencodedclient_credentials	client_idclient_secret
grant_typer  r  scope   r  r   timeout   r  r  r  r	   )astformat	TOKEN_URL
isinstancer  strliteral_evaljoinrequestspostr   jsonr  r    objectsfilterr  updater"   getr  r   )	r  r  urlscopes1r  r  r   r   resultr   r   r   get_new_tokenD  s*   *

zNinjaClient.get_new_tokenendpointc                 C   sV   t j| j d| |  |dd}|jdkr |   | ||S |jdkr)| S d S N/r  r  paramsr    r  r  r  BASE_URLr  r   r  
fetch_pager  r  r  r  rr   r   r   r  b  s   

zNinjaClient.fetch_page   
batch_sizec                 c   s   t | }g }|d dkr*| |i }|| t||kr#|V  g }|r(|V  d S |d dkrsd }	 |d |i}|rA|||d < | ||}||d g }|sRn|V  g }|dd }	|	rg|	|d }n|	sjn	 q3|rq|V  d S |d d	krd }
	 |d |d
|ddi}|
r|
||d < | ||}|sn|D ]}|| |d |d }
t||kr|V  g }qq||r|V  d S d S )Nr+   r   r   Tr   rl   r   r   r,   orderByr  r  rk   )NINJA_CONFIGr  extendlenr  append)r  r  r  cfgbatchr   r   r  r   cursor_conditionr,   recordr   r   r   stream_batchess  st   

zNinjaClient.stream_batchesNr  r   r   r   r  r  	PAGE_SIZEr  r  r  r  r  intr  r   r   r   r   r|  3  s    r|  c                   @   s>   e Zd Zdd Zdd ZdefddZdded	efd
dZdS )ConnectwiseClientc                 C   sB   || _ | j d | _|| _|| _| jd | _| jd d| _d S )Nr  site_urlr  z/v4_6_release/apis/3.0)r  r  r  r  r  rstripBase_urlr  r   r   r   r    s   zConnectwiseClient.__init__c                 C   r  r  r  r  r   r   r   r    r  zConnectwiseClient.headersr  c                 C   sF   t j| j d| |  |dd}|jdkrg S |jdkr!| S g S r  )r  r  r  r  r   r  r  r   r   r   r    s   

zConnectwiseClient.fetch_page'  r  c                 c   s    t | }g }|d dkrKd}	 |d |d ddi}|r"|||d < | ||}|r.|g kr/n|d7 }|| t||krC|V  g }q|rI|V  d S d S )	Nr+   r,   r  Tr   r   	pageinaterk   )CONNECTWISE_CONFIGr  r  r  r  r  r  r  r  r,   r  r   r   r   r   r    s0   
z ConnectwiseClient.stream_batchesNr  )	r   r   r   r  r  r  r  r  r  r   r   r   r   r    s
    r  c                   @   s>   e Zd Zdd Zdd Zdd Zdd Zd	d
 ZdddZdS )
ZohoClientc                 C   s:   || _ | j d | _|| _|| _| jd | _| jd | _d S )Nr  domainzohotype)r  r  r  r  r  	zoho_typer  r   r   r   r    s   zZohoClient.__init__c                 C   r  )NzZoho-oauthtoken r  r  r  r  r   r   r   r    r  zZohoClient.headersc                 C   s   dd l }d| j d}t| jd tr|| jd n| jd }d|}dtd d td d	 | jd d
}t	j
||d}|jdkrb| }|| _tjj| jdjt| jd |d| _| jS td)Nr   zhttps://accounts.zohoz/oauth/v2/tokenr  r  refresh_tokenZOHO	CLIENT_IDCLIENT_SECRETr  r  r  r  )r   r  r  r  r  r	   )r  r  r  r  r  r  r  r(   r  r  r  r   r  r    r  r  r  r  r"   r  r  r   )r  r  r  r  r  r   r   r  r   r   r   r  !  s&   *



zZohoClient.get_new_tokenc                 C   s^   d| j  | d}tj||  d}|jdkr |   | |S | }dd |dg D S )Nhttps://www.zohoapisz/organizationsr  r  c                 S      g | ]}|d  qS )organization_idr   )r.   or   r   r   
<listcomp>F      z0ZohoClient.get_organizations.<locals>.<listcomp>r   )r  r  r  r  r   r  get_organizationsr  )r  baser  r   r   r   r   r   r  >  s   

zZohoClient.get_organizationsc                 C   s  d}| j  dkrd }t| }d| j d| }n)| j  dkr.t| }d| j d| }n| j  dkrBt| }d| j d| }|i krvd	}| j  dkrRd }n| j  dkr_| d
}n| j  dkrk| d}|rtd|d i}nd	}tj|| 	 |dd}|j
dkr|   | ||S |j
dkr| }|r|d d u rt|d n|d }	|	d u r|}	||	|}
t|
tr|
gS |
S |S g S )NFzoho_crmr  z/crm/v2/zoho_inventoryz/inventory/v1/
zoho_booksz
/books/v3/T/inventory/v1	/books/v3r  r   r  r  r  r  r   )r  lowerZOHO_CRM_CONFIGr  ZOHO_INVENTORY_CONFIGZOHO_BOOKS_CONFIGr  r  r  r  r   r  r  r  	ZOHO_KEYSr  dict)r  r  r  
check_flagorganization_idsr  base_urlr  r   keyr   r   r   r   r  H  sN   


 
zZohoClient.fetch_pager  c                 c   s   | j  dkrt| }n$| j  dkrt| }| d}n| j  dkr/t| }| d}nd S g }|d dkrd}|d	 rg }|D ]S}	 |d |d|i}|rW|||d < | ||}	|d7 }|d d u rmt|d n|d }
|
d u rw|}
|	|
|}|	| |	dd d u rnt
||kr|V  g }qGqD|r|V  d S 	 |d |i}|r|||d < | ||}	|d7 }|d d u rt|d n|d }
|
d u r|}
|	|
|}|	| t
||kr|V  g }|	|d d d u rn|	|d d |d  snq|r|V  d S d S )Nr  r  r  r  r  r+   r,   r  r   Tr   r  rk   r   r   r   rl   )r  r  r  r  r  r  r  r  r  r  r  )r  r  r  r  r  r  r,   r  r  r   r  r   r   r   r   r  w  s~   
 
 
zZohoClient.stream_batchesN)r  )	r   r   r   r  r  r  r  r  r  r   r   r   r   r    s    
/r  c                   @   r{  )HubSpotClientzhttps://api.hubapi.comz%https://api.hubapi.com/oauth/v1/tokenr   c                 C   r}  r~  r  r  r   r   r   r    r  zHubSpotClient.__init__c                 C   r  r  r  r  r   r   r   r    r  zHubSpotClient.headersc                 C   s   dd l }t| jd tr|| jd n| jd }d|}ddi}d| jd | jd | jd d	}tj| j	||d
d}|j
dkr`| }|| _tjj| jdjt| jd |d| _| jS td)Nr   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r	   )r  r  r  r  r  r  r  r  r  r  r   r  r    r  r  r  r  r"   r  r  r   )r  r  r  r  r  r   r   r  r   r   r   r    s(   *

zHubSpotClient.get_new_tokenr  c                 C   s^   t j| j d| |  |dd}|jdkr |   | ||S |jdkr-| }|d S d S )Nr  r  r  r  r  r   r  )r  r  r  r  r   r   r   r   r    s   

zHubSpotClient.fetch_pager  r  c           
      c   s
   t | }g }|d dkr*| |i }|| t||kr#|V  g }|r(|V  d S |d dkrd }	 |d |i}|rA|||d < | ||}||d g }|sRn*|| t||krb|V  g }|dd }	|	rw|	|d	 |d
 }n|	szn	 q3|r|V  d S d S )Nr+   noner   Tr   rl   r   pagingr   r   )HUBSPOT_CONFIGr  r  r  r  r  
r  r  r  r  r  r   r   r  r   r  r   r   r   r  
  sL   

zHubSpotClient.stream_batchesNr  r  r   r   r   r   r    s    r  c                   @   F   e Zd Zdd Zdd Zdd Zdefdd	ZddedefddZ	dS )HubspotClientTokenc                 C   s.   || _ || _| jd | _|| _| jd | _d S N	api_tokenr  )r  r  r  r  r  r  r   r   r   r  B  s
   zHubspotClientToken.__init__c                 C   r  r  r  r  r   r   r   r  J  r  zHubspotClientToken.headersc                 C      d S Nr   r  r   r   r   r  P     z HubspotClientToken.get_new_tokenr  c           
      C   s   d}|i krd}ddi}| j d d| }tj||  |dd}|jdkr+td	|jd
krY| }|ddrC|d d d }n|d}|d }ddlm	}	 |	d|d|jdkrj| }|rh|d S |S d S )NFTr   r   r  r  r  r  r	   i  errorsr   r   requiredGranularScopes)PermissionDeniedzRequired scopes missing)r   required_scopesr  r   )
r  r  r  r  r  r   r   r  rest_framework.exceptionsr	  )
r  r  r  r  r  r  r   r   r
  r	  r   r   r   r  S  s>   



zHubspotClientToken.fetch_pager  r  c           
      c   s
   t | }g }|d dkr*| |i }|| t||kr#|V  g }|r(|V  d S |d dkrd }	 |d di}|rA|||d < | ||}||d g }|sRn*|| t||krb|V  g }|d	d }	|	rw|	|d
 |d }n|	szn	 q3|r|V  d S d S )Nr+   r  r   Tr     r   r   r  r   )r  r  r  r  r  r  r   r   r   r    sL   

z!HubspotClientToken.stream_batchesNr  
r   r   r   r  r  r  r  r  r  r  r   r   r   r   r  @  s    ,r  c                   @   sX   e Zd ZdZdZdd Zdd Zdd Zd	d
 Zdd Z	dd Z
ddedefddZdS )QuickbooksClientr   allc                 C   sN   || _ | j d | _|| _|| _d| _| jd | _| j d | _d| _|   d S )Nr  z9https://oauth.platform.intuit.com/oauth2/v1/tokens/bearerrealm_idr  )https://sandbox-quickbooks.api.intuit.com)	r  r  r  r  r  r  r  api_urlget_company_infor  r   r   r   r    s   zQuickbooksClient.__init__c                 C      d| j  dddS Nr  r  )r  r  r  r  r  r   r   r   r       
zQuickbooksClient.headersc                 C   s   dd l }ddd}dtd d td d | j| jd	}tj| j||d
d}|jdkrI| }|| _	t
jj| jdjt| j	d |d| _| jS td)Nr   r  r  )r  r  r  
QUICKBOOKSr  r  )r  r  r  r  r  r  r  r  r  r  r  r	   )r  r(   r  r  r  r  r  r   r  r  r    r  r  r  r  r"   r  r  r   r  r  r  r   r   r  r   r   r   r    s(   


zQuickbooksClient.get_new_tokenc                 C   s   dd l }| j d| j d}tj||  d}|jdkr:|j | di di gd di d		 | _
nd
| _
|j  	 | _d S )Nr   /v3/company/z6/query?query=select * from CompanyInfo&minorversion=69r  r  QueryResponser@   MetaData
CreateTimez
1970-01-01)datetimer  r  r  r  r  r   fromisoformatr  date	from_datenowto_date)r  r  r  r   r   r   r   r    s   
<z!QuickbooksClient.get_company_infoc              
   C   s   |  dv r-|  dks|  dkrdnd}| j d| j d| d| j d	| j d

}|S |dv rH| j d| j d| d| j d	| j d

}|S |dv r[| j d| j d| d}|S d }|S )N)rA   rB   balance/sheetrA   r#  BalanceSheetProfitAndLossr  z	/reports/z?start_date=z
&end_date=&minorversion=69)r2   r3   r4   r5   r6   r7   )r8   r9   r:   r;   r<   r=   r>   r?   z?minorversion=69)r  r  r  r   r"  )r  r  report_type
report_urlr   r   r   	build_url  s    **zQuickbooksClient.build_urlc                 C   s  |i krd}d}d}n
d}|d }|d }|  |}|rItj||  d}|jdkr8| }t|tr6|gS |S |jdkrG|   | 	||S g S d	| d
| d| }	| j
 d| j d|	 d}tj||  d}
|
jdkr|rzB|
 di t| g }|g ks|d kr|
 di |g }|g ks|d kr|
 di }|g ks|d kr|
 }W n   |
 }Y n|
 }t|tr|gS |S |
jdkr|   | 	||S g S )NTr  r  F	start_posmax_resultsr  r  zSELECT * FROM z STARTPOSITION z MAXRESULTS r  z/query?query=r&  r  )r)  r  r  r  r   r  r  r  r  r  r  r  r  
capitalize)r  r  r  r  r*  r+  r  r  r   queryr   	data_pager   r   r   r    sb   




 

zQuickbooksClient.fetch_pager  r  r  c                 c   s   t | }g }|d dkrd}d}	 ||d}| ||}|dvr| |}zB| di t| g }	|	g ks?|	d krg| di |g }	|	g ksS|	d krg| di }	|	g ksc|	d krg| }	W n   | }	Y t|	|k rxnM|
|7 }
||	 t||kr|V  g }|dd }|r||d	 |d
 }n#|sn!	 n| ||}|s|V  || t||kr|V  g }|V  q|r|V  d S d S d S )Nr+   r,   r   T)r*  r+  )r8   r9   r:   r;   r<   r=   r>   r?   r2   r3   r4   r5   r6   r7   r  r  r   r   )QUICKBOOKS_CONFIGr  r  r  r  r,  r  r  )r  r  r  r  r  r+  
start_pager  r   r.  r*  r  r   r   r   r   r  @  sd   
 


7
:zQuickbooksClient.stream_batchesNr  )r   r   r   r  SCOPESr  r  r  r  r)  r  r  r  r  r   r   r   r   r    s    ;r  c                   @   sH   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zddede	fddZ
dS )SalesforceClientc                 C   sL   || _ | j d | _|| _|| _| j d | _d| _| j d | _d| _t| _	d S )Nr  instance_urlz2https://login.salesforce.com/services/oauth2/tokenr  r  )
r  r  r  r  
domain_urlr  r  r  SALESFORCE_CONFIGr  r  r   r   r   r    s   
zSalesforceClient.__init__c                 C   s   dt | j ddS )Nr  r  )r  r  )r  r  r  r   r   r   r    s   zSalesforceClient.headersc                 C   s   dd l }| j}ddi}dtd d td d | jd}tj|||d	}|jd
krG| }|| _t	j
j| jdjt| jd |d| _| jS td)Nr   r  r  r  
SALESFORCEr  r  r  )r  r   r  r  r  r  r	   )r  r  r(   r  r  r  r   r  r  r    r  r  r  r  r"   r  r  r   r  r  r  r  r   r   r  r   r   r   r    s&   


zSalesforceClient.get_new_tokenc           	      C   s   | j  d| d}tj||  d}|jdkr | dg }n|jdkr/|   | ||S dS |}dd	 |D }d
|}|S )Nz/services/data/v53.0/sobjects/z	/describer  r  ru  r  Fc                 S   r  r  r   )r.   fieldr   r   r   r    r  z0SalesforceClient.get_columns.<locals>.<listcomp>, )	r4  r  r  r  r   r  r  r  r  )	r  r  r  describe_urlr  describe_dataru  column_namescolumns_stringr   r   r   get_columns  s   


zSalesforceClient.get_columnsc           
      C   s   d}| j  d}|i kr*d}| ||}| j  d}d| d| }| d| }n|d r4| d}n|d	 rA| d
|d	  }tj||  |dd}|jdkr_| }	|r[|	d S t	d|jdkrn|   | ||S t	d)NF/services/data/v53.0TSELECT  FROM z
/query/?q=ri   z/query/rj   r  r  r  r  r   r  zEndpoint Not Found)
r4  r?  r  r  r  r   r  r  r  r   )
r  r  r  r  r  r>  
soql_queryr  r  r   r   r   r   r    s6   

zSalesforceClient.fetch_pager  r  r  c                 c   s    | j | }g }| |i }| j d}d| d| }|d dkrjd }	 |d |i}	|r3||	|d < | ||	}
|
s<n'|
d	|}|| t||krR|V  g }|
|d d ra|
|d  }nnq%|rh|V  d S d S )
Nr@  rA  rB  r+   r,   Trk   rl   r   )r  r?  r4  r  r  r  r  )r  r  r  r  r  r>  r  rC  next_urlr  r   r   r   r   r   r    s:   

zSalesforceClient.stream_batchesNr  )r   r   r   r  r  r  r?  r  r  r  r  r   r   r   r   r2    s    r2  c                   @   r   )
JiraClientc                 C   sF   || _ || _| j d | _|| _| jd | _t| _| j d | _d| _d S )Nr  r  cloud_idz&https://auth.atlassian.com/oauth/token)	r  r  r  r  r  JIRA_CONFIGr  rF  r  r  r   r   r   r    s   
zJiraClient.__init__c                 C   r  r  r  r  r   r   r   r    r  zJiraClient.headersc                 C   s   dd l }ddi}d| jd | jd | jd d}tj| j|dd	}|jd
krK| }|| _| j| jd< t	j
j| jdjt| jd |d| _| jS dS )Nr   r  r  r  r  r  r  r  )r   r  r  rF  r  r  r  F)r  r  r  r  r  r  r   r  rF  r    r  r  r  r  r"   r  r  r  r   r   r   r  &  s&   
zJiraClient.get_new_tokenr  c           	      C   s   d| j  d}d}|i krd}ddd}tj| | |  |dd	}|jd
kr6|  }|r4| ||S dS |jdkr[t|d }| }|rY|g krQ|rQ|| }t	|t
rY|gS |S g S )Nz"https://api.atlassian.com/ex/jira/z/rest/api/3/FTr   r   )r   r   <   r  r  r  )rF  r  r  r  r   r  r  	JIRA_KEYSr  r  r  )	r  r  r  r  r  r  r   r  r   r   r   r   r  B  s:   



zJiraClient.fetch_pager  r  c           
      c   s    | j | }g }|d dkrud}	 |d |d i}|r!|||d < | ||}t|tr5|s/n?|| n9|r=||d  s>n0t|d }|rM|||}	n|}	||	 ||d	  sa||d
 7 }nnt||krm|V  g }q|rs|V  d S d S )Nr+   r,   r   Tr   r   rk   r   rl   total)r  r  r  listr  rI  r  r  
r  r  r  r  r  r,   r  r   r  r   r   r   r   r  h  sD   



"zJiraClient.stream_batchesNr  r  r   r   r   r   rE    s    
&rE  c                   @   r   )
Pax8Clientc                 C   s:   || _ || _| j d | _|| _| jd | _t| _d| _d S )Nr  r  zhttps://api.pax8.com/v1/token)r  r  r  r  r  PAX8_CONFIGr  r  r  r   r   r   r    s   
zPax8Client.__init__c                 C   r  r  r  r  r   r   r   r    r  zPax8Client.headersc                 C   s   dd l }ddd}dd| jd | jd d}tj| j||d	d
}|jdkrD| }|| _tj	j
| jdjt| jd |d| _| jS dS )Nr   r  )r  acceptr  zhttps://api.pax8.comr  r  )r  audiencer  r  rH  )r  r  r  r  r  r  r  F)r  r  r  r  r  r   r  r  r    r  r  r  r  r"   r  r  r  r   r   r   r    s&   
zPax8Client.get_new_tokenr  c                 C   s   | j  d}d}|i krd}ddd}tj| | |  |dd}|jd	kr5|  }|r3| ||S dS |jd
krR| }|rP|g krH|d }t|t	rP|gS |S g S )N/v1/FTr   r  )r   r,   rH  r  r  r  content)
r  r  r  r  r   r  r  r  r  r  )r  r  r  r  r  r  r   r   r   r   r   r    s6   



zPax8Client.fetch_pager  r  c                 c   s    | j | }g }|d dkrYd}	 |d |d i}|r!|||d < | ||}|s*n(||d  t||kr<|V  g }|dd	 |dd
 d krMn|d7 }q|rW|V  d S d S )Nr+   r,   r  Tr   r   rk   rR  
totalPagesnumber)r  r  r  r  r  r  r   r   r   r    s2   
 zPax8Client.stream_batchesNr  r  r   r   r   r   rM    s    	%rM  c                   @   r   )TallyClientc                 C   s4   || _ || _| jd | _|| _| jd | _t| _d S r  )r  r  r  r  r  TALLY_CONGIGr  r  r   r   r   r    s   
zTallyClient.__init__c                 C   r  r  r  r  r   r   r   r     r  zTallyClient.headersc                 C   r  r  r   r  r   r   r   r  &  r  zTallyClient.get_new_tokenr  c                 C   s   d}|i krd}t j| j | |  d}nt j| j | |  |d}|jdkrOt|d }| }|rM|dkrM|g krE|rE|| }t|trM|gS |S g S )NFTr  r  r  r  r   )	r  r  r  r  r   
TALLY_KEYSr  r  r  )r  r  r  r  r  r  r   r   r   r   r  )  s.   

zTallyClient.fetch_pager  r  c           
      c   s   | j | }g }|d dkrcd}	 |d |d i}|r!|||d < | ||}|s*n2|d7 }t||d }|d u r<|}|||}	||	 ||d	 d
sPnt||kr[|V  g }q|ra|V  d S |d dkri }| ||}|| t||kr|V  g }|r|V  d S d S )Nr+   r,   r  Tr   r   rk   r   rl   Fr   )r  r  rX  r  r  r  r  rL  r   r   r   r  J  sL   



zTallyClient.stream_batchesNr  r  r   r   r   r   rU    s    !rU  c                   @   s.   e Zd Zdd Zdd Zi fdefddZdS )	GoogleSheetClientc                 C   s4   || _ | j d | _|| _|| _d| _| j d | _d S )Nr  #https://oauth2.googleapis.com/tokenr  )r  r  r  r  r  r  r  r   r   r   r    s   zGoogleSheetClient.__init__c                 C   s|   dd l }ddd| jd d}tj| j|d}|jdkr:| }|| _tjj	| j
djt| jd	 |d
| _| jS td)Nr   r  H651576155260-4v9d0nii8qbtun6ntnif5n05cm9l3u6u.apps.googleusercontent.com#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r  )r  r  r  r  r  r	   )r  r  r  r  r  r   r  r    r  r  r  r  r"   r  r  r   )r  r  r   r   r  r   r   r   r    s    
zGoogleSheetClient.get_new_tokenr  c              
   C   s  ddl m} ddlm} z|| j| jdddd}W n ty5 } z|   | ||W  Y d }~S d }~ww |dd	|d
}z|	 j
|d }W n tye } z|   | ||W  Y d }~S d }~ww |
dg }	|
di 
ddpw|}
|	d }|d d }|d 
di 
dd}|d 
di 
dd}|	  j
|| d| d }|
dg }dd |D }t|tr|gS |S )Nr   )Credentials)buildz=https://app.analytify.ai/analytify/datasources/google-sheets/r[  r\  )tokenr  	token_urir  r  sheetsv4r  )spreadsheetId
propertiestitle gridPropertiesrowCountcolumnCountz!1:)rd  ranger   c                 S   s   g | ]}t |r|qS r   )any)r.   rowr   r   r   r    s    z0GoogleSheetClient.fetch_page.<locals>.<listcomp>)google.oauth2.credentialsr]  googleapiclient.discoveryr^  r  r  	Exceptionr  r  spreadsheetsr  executer   r  r  )r  r  r  r]  r^  credsr/   sheets_servicespreadsheet_metadatara  spreadsheet_titlesheet
sheet_namemax_rowsmax_colsr  r   non_empty_rowsr   r   r   r    sH   

zGoogleSheetClient.fetch_pageN)r   r   r   r  r  r  r  r   r   r   r   rY  }  s    rY  c                   @   sN   e Zd ZdZdZdd Zdd Zdd Zd	efd
dZ	dd	ede
fddZdS )HalopsaClientr   r  c                 C   s.   || _ | j d | _|| _|| _| jd | _d S )Nr  r  )r  r  r  r  r  r  r   r   r   r    s
   zHalopsaClient.__init__c                 C   r  r  r  r  r   r   r   r    r  zHalopsaClient.headersc                 C   s   dd l }d| j}ddi}d| jd | jd | jd}tj|||d	d
}|jdkrI| }|| _	t
jj| jdjt| j	d |d| _| jS td)Nr   z{}/auth/tokenr  r  r  r  r  r  r  r  r  r  r  r  r	   )r  r  r  r  r1  r  r  r   r  r  r    r  r  r  r  r"   r  r  r   r7  r   r   r   r    s&   
zHalopsaClient.get_new_tokenr  c                 C   sz   t j| j d| |  |dd}|jdkr |   | ||S |jdkr;t|d }| }|g kr9|r9|| }|S g S )Nz/api/r  r  r  r  )	r  r  r  r  r   r  r  Halopsa_keysr  )r  r  r  r  r  r   r   r   r   r    s"   

zHalopsaClient.fetch_pager  r  c                 c   s    t | }g }|d dkrKd}	 |d |d ddi}|r"|||d < | ||}|r.|g kr/n|d7 }|| t||krC|V  g }q|rI|V  d S |d d	krn| |i }|| t||krg|V  g }|rl|V  d S d S )
Nr+   r,   r  Tr   r   r  rk   r   )HALOPSA_CONFIGr  r  r  r  r   r   r   r    sB   

zHalopsaClient.stream_batchesNr  )r   r   r   r  r1  r  r  r  r  r  r  r  r   r   r   r   r|    s    r|  c                   @   s0   e Zd ZddedefddZdd Zdd	 Zd
S )
Normalizerr  
root_tablepk_fieldc                 C   s   || _ || _d S r  )r  r  )r  r  r  r   r   r   r  1  s   
zNormalizer.__init__c              	   C   sn   t t}t }|D ])}t|tst|trtt }n|| jp%tt }| j|| j	|d ||d q	||fS )N)objtablepk_valueparenttables	relations)
r   rK  setr  r  r   r  r  _walkr  )r  r   r  r  r  root_idr   r   r   	normalize5  s   	zNormalizer.normalizec                 C   s  | j |i}dd l}|dd|}|rF|\}	}
|dd|	}	|	 d}|
||< |	dd|dd| j dd|ddf}||vrF|| | D ]\}}t|trmtt	 }| d| }| j
|||||f||d qJt|tr| d| }tt	}|D ]J}t|trtt	 }| j
|||||f||d q| j || d|d|i}||dd|dd| j dd| dddf || | qqJ|||< qJ|| | d S )	Nr   z[^a-zA-Z0-9_]+__id-__)r  r  r  value)r  resubreplaceaddr   r  r  r  r   r  rK  r  )r  r  r  r  r  r  r  rm  r  parent_table	parent_pkfk_fieldrelationkvchild_pkchild_tableitem	child_rowr   r   r   r  I  sn   








	






zNormalizer._walkNr  )r   r   r   r  r  r  r  r   r   r   r   r  0  s    r  c                   @   s>   e Zd Zdd Zdd Zdi fddZdd	ed
efddZdS )GoogleAnalyticClientc                 C   s   || _ || _|| _d S r  )r  r  r  r  r   r   r   r    s   
zGoogleAnalyticClient.__init__c                 C   sL   | j d | j d | j d | j d dd| j d | j d d	d
d| j d d
S )Nr+   
project_idprivate_key_idprivate_keyz\n
client_emailr  z)https://accounts.google.com/o/oauth2/authrZ  z*https://www.googleapis.com/oauth2/v1/certsclient_x509_cert_url)
r+   r  r  r  r  r  auth_urir`  auth_provider_x509_cert_urlr  )r  r  r  r   r   r   get_service_info  s   z%GoogleAnalyticClient.get_service_infoNc              
      st  ddl m}m}m m ddlm} ddlm} z
|j	
|  }W n   g  Y S ||d}| jd }	| jdp=g d}
| jd	pGd
dg}|d|	  fdd|
D fdd|D |dddgd}z||}W n ty } zW Y d }~dS d }~ww g }|jD ]1}i }t|jD ]\}}|j| j||j< qt|jD ]\}}t|j| j||j< q|| q|S )Nr   )RunReportRequest	DateRange	DimensionMetric)BetaAnalyticsDataClient)service_accountrc  property_id
dimensions)r  countrydeviceCategorymetricsactiveUsersnewUserszproperties/c                       g | ]} |d qS r8  r   )r.   d)r  r   r   r        z3GoogleAnalyticClient.fetch_page.<locals>.<listcomp>c                    r  r  r   )r.   m)r  r   r   r    r  z
2015-08-14today)
start_dateend_date)propertyr  r  date_rangesF)"google.analytics.data_v1beta.typesr  r  r  r  google.analytics.data_v1betar  google.oauth2r  r]  from_service_account_infor  r  r  
run_reportrp  rows	enumeratedimension_headersdimension_valuesr  r  metric_headersr  metric_valuesr  )r  r  parmasr  r  r  r  r  clientr  r  r  request_datar   r/   r  rm  row_datai	dimensionmetricr   )r  r  r   r    s@   


zGoogleAnalyticClient.fetch_pager  r  r  c                 c   s    |  ||V  d S r  )r  )r  r  r  r   r   r   r    s   z#GoogleAnalyticClient.stream_batchesr  )	r   r   r   r  r  r  r  r  r  r   r   r   r   r    s
     r  )Session)HTTPAdapter)Retry)ConnectionErrorTimeoutSSLErrorRequestExceptionc                   @   sN   e Zd Zdd Zdd Zdd Zdd Zd	efd
dZdd	ede	fddZ
dS )ShopifyClientc                 C   s:   || _ || _| jd | _d| _|| _| jd | _t| _d S )Nr  z2024-01r  )r  r  r  api_versionr  r  BAMBOOHR_CONFIGr  r  r   r   r   r    s   
zShopifyClient.__init__c                 C   s   | j  ddS )Nr  )zX-Shopify-Access-Tokenr  r  r  r   r   r   r    s   zShopifyClient.headersc                 C   r  r  r   r  r   r   r   r    r  zShopifyClient.get_new_tokenc              	   C   sh   t | dr| jr| jS tddddg ddgdd}t|ddd	}t }|d
| |d| || _|S )Nsession   g      ?)i  r  i  i  i  GETF)rJ  connectreadbackoff_factorstatus_forcelistallowed_methodsraise_on_status
   )max_retriespool_connectionspool_maxsizezhttps://zhttp://)hasattrr  r  r  r  mount)r  retryadapterr  r   r   r   _get_session  s*   
zShopifyClient._get_sessionr  c              
   C   s   d}|i krd}ddd}d}|dkr!| j  d| j d	| d
}n| j  d| j d	| d
}|  }z|j||  |dd}W n tttfyY } ztd| d| |d }~ww |j	dkru|
 }	|rs|	| }
t|
trq|
gS |
S |S |j	dkr|tg S )NFTrl  r   r   r   2   r   z/admin/api/r  z.json)r    r  z"Network/TLS failure while calling z: r  )r  r  r  r  r  r  r  r  RuntimeErrorr   r  r  r  PermissionError)r  r  r  r  r   next_page_urlr  r  r/   r   r   r   r   r   r  		  sJ   



zShopifyClient.fetch_pager  r  c                 c   s
   g }d }	 ddd}|r|dd}|  ||}|g krna| }|jdd }|rhddlm}	m}
 |jdd	}t|d
krK|d dd }n	|d dd }|	|}|
|j	}|dd 
d}nn|| }|| t||kr}|V  g }q|r|V  d S )NTrl  r   r  )	page_infor   linkr   )urlparseparse_qs,   ;r  r  >)r  r  r  r  urllib.parser  r  splitr  r-  stripr  )r  r  r  r  r  r  r   response_datar  r  r  r   r  
parsed_urlr   r   r   r  :	  sF   

)zShopifyClient.stream_batchesNr  )r   r   r   r  r  r  r  r  r  r  r  r   r   r   r   r    s    
1r  c                   @   r   )BambooHrClientc                 C   sf   || _ || _| jd | _t| j d  | _|| _| jd | _	| jd d | j	 | _
t| _d S )Nr  z:xr  r  r  )r  r  r_  base64	b64encodeencodedecoder  r  r  r  r  r  r  r   r   r   r  n	  s   
zBambooHrClient.__init__c                 C   r  )NzBasic r  r  r  r  r   r   r   r  y	  r  zBambooHrClient.headersc                 C   r  r  r   r  r   r   r   r  	  r  zBambooHrClient.get_new_tokenr  c                 C   s   d}|i krd}ddd}t j| j d| |  |d}|jdkrB| }|r@t|d }|g kr@|r8|| }t|tr@|gS |S g S )	NFTr  r   )r,   r   rQ  rW  r  )	r  r  r  r  r   r  BAMBOOHR_KEYSr  r  )r  r  r  r  r  r   r  r   r   r   r  	  s,   

zBambooHrClient.fetch_pager  r  c           
      c   s    | j | }g }|d dkrsd}	 |d |d i}|r!|||d < | ||}|s*nB|d7 }t|d }|r<||}	n|}	||	 t||krN|V  g }|dd rj|dd	 |dd
 d kren|d7 }nnq|rq|V  d S d S )Nr+   r,   r  Tr   r   rk   
paginationtotal_pagescurrent_page)r  r  r  r  r  r  rL  r   r   r   r  	  s@   

 
!zBambooHrClient.stream_batchesNr  r  r   r   r   r   r  l	  s    r  c                   @   r   )	DbtClientc                 C   s@   || _ || _| jd | _|| _| jd | _| jd | _t| _d S )Nr  r  
account_id)r  r  r  r  r  r  
DBT_CONFIGr  r  r   r   r   r  	  s   
zDbtClient.__init__c                 C   r  r  r  r  r   r   r   r  	  r  zDbtClient.headersc                 C   r  r  r   r  r   r   r   r  	  r  zDbtClient.get_new_tokenr  c                 C   s   d}|i krd}ddd}|dkr| j d d}n| j d d| j d| }tj||  |d	d
}|jdkrO| }|rM|d}t|t	rM|gS |S |S )NFTr   r   )r   r   r   r  z/api/v2/accounts/r  r  r  r   )
r  r  r  r  r  r  r   r  r  r  )r  r  r  r  r  r  r   r   r   r   r  	  s.   


zDbtClient.fetch_pager  r  c           
      c   s    | j | }g }d}|d dkr_d}	 |d |d i}|r#|||d < | ||}|d	ds0n(||d	g  t||krD|V  g }|d
d}	|	d |k rSn||7 }q|r]|V  d S d S )Nr   r+   r,   r   Tr   r   rk   r   extrar  total_count)r  r  r  r  r  )
r  r  r  r  r  r   r,   r  r   r  r   r   r   r  
  s6   
zDbtClient.stream_batchesNr  r  r   r   r   r   r
  	  s    	!r
  df	ch_schemareturnc                 C   s   |  D ]s\}}d|v rtj| | ddd| |< nId|v r+tj| | dd| |< n9d|v r;tj| | dd| |< n)d|v rMtj| | ddjj| |< nd|v r[| | t| |< n	| | t| |< |	d	rw| | 
| |  d | |< q| S )
NIntcoercer  Int64FloatDateTimeDateBoolNullable)r   pd
to_numericastypeto_datetimedtr  boolr  
startswithwherenotnar  r  colch_typer   r   r   !sanitize_dataframe_for_clickhouse0
  s    
r'  )create_enginec                 C   s   |   } dd | jD | _| S )Nc                 S   s   g | ]	}t d d|qS )[^a-zA-Z0-9_]r  r  r  )r.   cr   r   r   r  U
  s    z%normalize_columns.<locals>.<listcomp>)copycolumns)r  r   r   r   normalize_columnsR
  s
   r.  c                 C   s   |  D ]Y\}}|| jvrqd|v r!tj| | ddd| |< qd|v r1tj| | dd| |< qd|v rDtj| | ddd| |< qd|v rTtj| | dd| |< q| | t| |< q| t| d	} | S )
z-
    ch_schema: {column: ClickHouseType}
    r  r  r  r  r  UIntUInt64r  N)	r   r-  r  r  r  r  r  r"  notnullr$  r   r   r   cast_df_to_clickhouse_schema\
  s   
r2  c                   @   sj   e Zd ZdddZdddZdd Zd	d
 ZddefddZdd Z	dd Z
deeee f fddZdS )
Clickhouser)   c              	   C   s   t jttttti ddddddddddddd	dd
dddddddddddddddddddd| _d S )Ndate_time_input_formatbest_effortinput_format_null_as_defaultr  async_insertwait_for_async_insertinput_format_try_infer_dates input_format_try_infer_datetimes0input_format_try_infer_datetimes_only_datetime644input_format_csv_use_best_effort_in_schema_inference.enable_extended_results_for_datetime_functionsreceive_timeoutiX  http_receive_timeoutrH  receive_data_timeout_msr  async_insert_busy_timeout_msi  insert_null_as_defaultmax_columns_to_readr  max_insert_block_sizei max_query_sizei )hostportusernamepassworddatabasesettings)clickhouse_connect
get_clientclickhouse_hostclickhouse_portclickhouse_usernameclickhouse_passwordclickhouse_databaser  )r  rJ  r   r   r   r  y
  s>   	
zClickhouse.__init__r   c              	   C   sv   t | |d  }tdt||D ]&}||||  }dd |D }	d| d| dd|	 d}
| j|
 qd S )	Nr   c                 S   s*   g | ]\}}d t dd| d| qS )zADD COLUMN IF NOT EXISTS `r)  r  ` r*  r.   r+  tr   r   r   r  
  s    z7Clickhouse.ensure_remaining_columns.<locals>.<listcomp>z
            ALTER TABLE ``.`z`
            r:  z
            )rK  r   rk  r  r  r  command)r  rJ  r  schemar   r  r   r  r  altersr-  r   r   r   ensure_remaining_columns
  s   z#Clickhouse.ensure_remaining_columnsc                 C   sn   t | }|d d }ddd |D }d| d| d| d}t|dkr/| ||| | j| d S )	Nr   z,
c                 s   s0    | ]\}}d t dd| d| dV  qdS )`r)  r  rS  r  Nr*  rT  r   r   r   	<genexpr>
  s   . z*Clickhouse.ensure_table.<locals>.<genexpr>z%
        CREATE TABLE IF NOT EXISTS `rV  z` (
            zG
        )
        ENGINE = MergeTree
        ORDER BY tuple()
        )rK  r   r  r  rZ  r  rW  )r  rJ  r  rX  r   first_batchcolsr-  r   r   r   ensure_table
  s   zClickhouse.ensure_tablec                 C   sJ   |  D ]\}}tdd|}| jd| d| d| d| d	 qd S )Nr)  r  z
                ALTER TABLE `rV  z,`
                ADD COLUMN IF NOT EXISTS `rS  z
                )r   r  r  r  rW  )r  rJ  r  new_columnsr%  dtyper   r   r   evolve_schema
  s   zClickhouse.evolve_schemaNr  c                    sb   t |}t|}t||| }|t |d }dd l | fdd}| jj	|||d d S )Nr   c                    s   t | t jfrt| S | S r  )r  r   bool_r  )xnpr   r   <lambda>
  s    z(Clickhouse.copy_insert.<locals>.<lambda>)rJ  r  r  )
r  	DataFramer.  r2  r"  r1  numpymapr  	insert_df)r  r  rJ  	dataframeschema_cacher  r   re  r   copy_insert
  s    


zClickhouse.copy_insertc                 C      | j d| d d S )NzCreate Database ""r  r-  r  database_namer   r   r   Create_database
     zClickhouse.Create_databasec                 C   ro  )NzDrop DAtabase if Exists "z" rq  rr  r   r   r   drop_database
  ru  zClickhouse.drop_databaser  c                 C   s   |  D ]=\}}t|}t|}t|}t|}t|}t|}t	dd|}d| d| d| d}| j
| | ||| qd S )Nr)  r  z&
            CREATE or Replace Table "z".z (

                zQ
            ) ENGINE = MergeTree()
            ORDER BY tuple()
                )r   r  rh  r.  r$   r%   r&   r'   r  r  r  r-  rn  )r  rJ  r  r  r  rl  rX  create_table_queryr   r   r   create_tables
  s$   
zClickhouse.create_tables)r)   )r   r   r  )r   r   r   r  rZ  r_  rb  r  rn  rt  rv  r   r   r  rx  r   r   r   r   r3  w
  s    

+r3  )fr  collectionsr   rest_framework.responser   rest_frameworkr   r  r   r   r   r   rest_framework.viewsr   r   uuidr   ior   typingr   r   Connections.modelsr    psycopg2psycopg2.extrasr!   Service.utilsr"   r#   Connections.utilsr$   r%   r&   r'   Datamplify.settingsr(   rN  rO  rP  rQ  rR  r/  r5  rN  rG  r  rI  rV  rX  r  r  r  r  r  r  r  r~  r}  r  r  SHOPIFY_CONFIGr|  r  r  r  r  r  r2  rE  rM  rU  rY  r|  r  r  r  r  requests.adaptersr  urllib3.util.retryr  requests.exceptionsr  r  r  r  r  r  r
  pandasr  rh  r  r'  rL  
sqlalchemyr(  r  r.  r2  r3  r   r   r   r   <module>   s  
(

	`-;  J :wy Q  |gArb; d^
