o
    ;Did                     @   s  d dl mZmZmZ d dlZd dlZd dlmZ d dlm	Z	m
Z
mZmZ d dlmZ d dlmZmZmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZ d dlmZ 		d9ddZee_G dd deZdd Z		d:ddZ e e_ G dd deZ!dd Z"G dd deZ#dd Z$dd  Z%e%e_%		d;d!d"Z&e&e_&		d;d#d$Z'e'e_'		d;d%d&Z(e(e_(		d;d'd(Z)e)e_)		d9d)d*Z*e*e_*G d+d, d,eZ+d-d. Z,d/d0 Z-e-e_-G d1d2 d2e.Z/		d9d3d4Z0e0e_0G d5d6 d6eZ1d7d8 Z2dS )<    )absolute_importprint_functiondivisionN)OrderedDict)nextstring_typesreduce	text_type)ArgumentError)Tableiterpeek
rowgroupby)values)nrows)sort	mergesort)cut)distinctFTc              
   C      t | |||||||dS )a  
    Group rows under the given key then apply `reducer` to produce a single
    output row for each input group of rows. E.g.::
    
        >>> import petl as etl
        >>> table1 = [['foo', 'bar'],
        ...           ['a', 3],
        ...           ['a', 7],
        ...           ['b', 2],
        ...           ['b', 1],
        ...           ['b', 9],
        ...           ['c', 4]]
        >>> def sumbar(key, rows):
        ...     return [key, sum(row[1] for row in rows)]
        ...
        >>> table2 = etl.rowreduce(table1, key='foo', reducer=sumbar,
        ...                        header=['foo', 'barsum'])
        >>> table2
        +-----+--------+
        | foo | barsum |
        +=====+========+
        | 'a' |     10 |
        +-----+--------+
        | 'b' |     12 |
        +-----+--------+
        | 'c' |      4 |
        +-----+--------+
    
    N.B., this is not strictly a "reduce" in the sense of the standard Python
    :func:`reduce` function, i.e., the `reducer` function is *not* applied 
    recursively to values within a group, rather it is applied once to each row 
    group as a whole.
    
    See also :func:`petl.transform.reductions.aggregate` and
    :func:`petl.transform.reductions.fold`.
    
    )header	presorted
buffersizetempdircache)RowReduceView)tablekeyreducerr   r   r   r   r    r   R/var/www/Datamplify/venv/lib/python3.10/site-packages/petl/transform/reductions.py	rowreduce   s   
(r    c                   @   s"   e Zd Z		dddZdd ZdS )	r   NFTc	           	      C   s6   |r|| _ n
t|||||d| _ || _|| _|| _d S Nr   r   r   )sourcer   r   r   r   )	selfr#   r   r   r   r   r   r   r   r   r   r   __init__E      
zRowReduceView.__init__c                 C      t | j| j| j| jS N)iterrowreducer#   r   r   r   r$   r   r   r   __iter__P      zRowReduceView.__iter__NFNNT__name__
__module____qualname__r%   r+   r   r   r   r   r   C   
    
r   c                 c   sH    |d u rt | \}} t|V  t| |D ]\}}t|||V  qd S r(   )r   tupler   )r#   r   r   r   rowsr   r   r   r)   T   s   
r)   valuec	           	      C   sX   t |rt| ||||||||d	S |du st|tttfr(t| ||||||dS td)a  Apply aggregation functions.
    E.g.::

        >>> import petl as etl
        >>>
        >>> table1 = [['foo', 'bar', 'baz'],
        ...           ['a', 3, True],
        ...           ['a', 7, False],
        ...           ['b', 2, True],
        ...           ['b', 2, False],
        ...           ['b', 9, False],
        ...           ['c', 4, True]]
        >>> # aggregate whole rows
        ... table2 = etl.aggregate(table1, 'foo', len)
        >>> table2
        +-----+-------+
        | foo | value |
        +=====+=======+
        | 'a' |     2 |
        +-----+-------+
        | 'b' |     3 |
        +-----+-------+
        | 'c' |     1 |
        +-----+-------+

        >>> # aggregate whole rows without a key
        >>> etl.aggregate(table1, None, len)
        +-------+
        | value |
        +=======+
        |     6 |
        +-------+

        >>> # aggregate single field
        ... table3 = etl.aggregate(table1, 'foo', sum, 'bar')
        >>> table3
        +-----+-------+
        | foo | value |
        +=====+=======+
        | 'a' |    10 |
        +-----+-------+
        | 'b' |    13 |
        +-----+-------+
        | 'c' |     4 |
        +-----+-------+

        >>> # aggregate single field without a key
        >>> etl.aggregate(table1, None, sum, 'bar')
        +-------+
        | value |
        +=======+
        |    27 |
        +-------+

        >>> # alternative signature using keyword args
        ... table4 = etl.aggregate(table1, key=('foo', 'bar'),
        ...                        aggregation=list, value=('bar', 'baz'))
        >>> table4
        +-----+-----+-------------------------+
        | foo | bar | value                   |
        +=====+=====+=========================+
        | 'a' |   3 | [(3, True)]             |
        +-----+-----+-------------------------+
        | 'a' |   7 | [(7, False)]            |
        +-----+-----+-------------------------+
        | 'b' |   2 | [(2, True), (2, False)] |
        +-----+-----+-------------------------+
        | 'b' |   9 | [(9, False)]            |
        +-----+-----+-------------------------+
        | 'c' |   4 | [(4, True)]             |
        +-----+-----+-------------------------+

        >>> # alternative signature using keyword args without a key
        >>> etl.aggregate(table1, key=None,
        ...                 aggregation=list, value=('bar', 'baz'))
        +-----------------------------------------------------------------------+
        | value                                                                 |
        +=======================================================================+
        | [(3, True), (7, False), (2, True), (2, False), (9, False), (4, True)] |
        +-----------------------------------------------------------------------+

        >>> # aggregate multiple fields
        ... from collections import OrderedDict
        >>> import petl as etl
        >>>
        >>> aggregation = OrderedDict()
        >>> aggregation['count'] = len
        >>> aggregation['minbar'] = 'bar', min
        >>> aggregation['maxbar'] = 'bar', max
        >>> aggregation['sumbar'] = 'bar', sum
        >>> # default aggregation function is list
        ... aggregation['listbar'] = 'bar'
        >>> aggregation['listbarbaz'] = ('bar', 'baz'), list
        >>> aggregation['bars'] = 'bar', etl.strjoin(', ')
        >>> table5 = etl.aggregate(table1, 'foo', aggregation)
        >>> table5
        +-----+-------+--------+--------+--------+-----------+-------------------------------------+-----------+
        | foo | count | minbar | maxbar | sumbar | listbar   | listbarbaz                          | bars      |
        +=====+=======+========+========+========+===========+=====================================+===========+
        | 'a' |     2 |      3 |      7 |     10 | [3, 7]    | [(3, True), (7, False)]             | '3, 7'    |
        +-----+-------+--------+--------+--------+-----------+-------------------------------------+-----------+
        | 'b' |     3 |      2 |      9 |     13 | [2, 2, 9] | [(2, True), (2, False), (9, False)] | '2, 2, 9' |
        +-----+-------+--------+--------+--------+-----------+-------------------------------------+-----------+
        | 'c' |     1 |      4 |      4 |      4 | [4]       | [(4, True)]                         | '4'       |
        +-----+-------+--------+--------+--------+-----------+-------------------------------------+-----------+

        >>> # aggregate multiple fields without a key
        >>> etl.aggregate(table1, None, aggregation)
        +-------+--------+--------+--------+--------------------+-----------------------------------------------------------------------+--------------------+
        | count | minbar | maxbar | sumbar | listbar            | listbarbaz                                                            | bars               |
        +=======+========+========+========+====================+=======================================================================+====================+
        |     6 |      2 |      9 |     27 | [3, 7, 2, 2, 9, 4] | [(3, True), (7, False), (2, True), (2, False), (9, False), (4, True)] | '3, 7, 2, 2, 9, 4' |
        +-------+--------+--------+--------+--------------------+-----------------------------------------------------------------------+--------------------+

    If `presorted` is True, it is assumed that the data are already sorted by
    the given key, and the `buffersize`, `tempdir` and `cache` arguments are 
    ignored. Otherwise, the data are sorted, see also the discussion of the 
    `buffersize`, `tempdir` and `cache` arguments under the
    :func:`petl.transform.sorts.sort` function.

    If `key` is None, sorting is not necessary.

    )aggregationr5   r   r   r   r   fieldN)r6   r   r   r   r   z;expected aggregation is callable, list, tuple, dict or None)callableSimpleAggregateView
isinstancelistr3   dictMultiAggregateViewr
   )	r   r   r6   r5   r   r   r   r   r7   r   r   r   	aggregate]   s   ~r>   c                   @   s,   e Zd ZeddddddfddZdd ZdS )	r9   NFTr5   c
           
      C   sD   |s|d u r
|| _ n
t|||||d| _ || _|| _|| _|	| _d S r!   )r   r   r   r6   r5   r7   )
r$   r   r   r6   r5   r   r   r   r   r7   r   r   r   r%      s   
zSimpleAggregateView.__init__c                 C   s   t | j| j| j| j| jS r(   )itersimpleaggregater   r   r6   r5   r7   r*   r   r   r   r+      s   zSimpleAggregateView.__iter__)r/   r0   r1   r;   r%   r+   r   r   r   r   r9      s    
r9   c                 c   s$   |t kr|d urdd }t|ttfrt |dkr|d }t|ttfr-t||f }nt|r6d|f}n|d u r>|f}n||f}|V  t|ttfrct| ||D ]\}}t|||f V  qRd S |d u r~|t krst| fV  d S |t| |fV  d S t| ||D ]\}}|||fV  qd S )Nc                 S   s   t dd | D S )Nc                 s   s    | ]}d V  qdS )   Nr   ).0_r   r   r   	<genexpr>  s    z8itersimpleaggregate.<locals>.<lambda>.<locals>.<genexpr>)sum)gr   r   r   <lambda>  s    z%itersimpleaggregate.<locals>.<lambda>r@   r   r   )lenr:   r;   r3   r8   r   r   r   )r   r   r6   r5   r7   outhdrkgrpr   r   r   r?     s0   
r?   c                   @   s*   e Zd Z		d
ddZdd Zdd	 ZdS )r=   NFTc           	      C   s   |s|d u r
|| _ n
t|||||d| _ || _|d u r!t | _d S t|ttfr>t | _|D ]}|dd  | j|d < q.d S t|trH|| _d S t	d| )Nr"   r@   r   z;expected aggregation is None, list, tuple or dict, found %r)
r#   r   r   r   r6   r:   r;   r3   r<   r
   )	r$   r#   r   r6   r   r   r   r   tr   r   r   r%   '  s(   

zMultiAggregateView.__init__c                 C      t | j| j| jS r(   )itermultiaggregater#   r   r6   r*   r   r   r   r+   =     zMultiAggregateView.__iter__c                 C   s   || j |< d S r(   )r6   )r$   r   r5   r   r   r   __setitem__@  s   zMultiAggregateView.__setitem__r-   )r/   r0   r1   r%   r+   rO   r   r   r   r   r=   %  s    
r=   c                 #   s\   t | }t| }t| t g|}|D ]V}|| }t|r)d |f||< qt|tr5|t	f||< qt
|dkrKt|d trK|d t	f||< qt
|dkr`t|d r`d |d f||< qt
|dkrgqtd||f t|t	tfr{t	|}nt|rdg}n
|d u rg }n|g}|D ]}|| qt|V  |d u rt|dd }nt||}|D ]|\}}	t	|	}	t|t	tfrt	|}
n
|d u rg }
n|g}
|D ]V}|| \}}|d u r||	}|
| qt|t	tfr fdd	|D }tj| fd
d|	D }||}|
| qψ |fdd|	D }||}|
| qt|
V  qd S )Nr@   r      zinvalid aggregation: %r, %rr   c                 S   s   d S r(   r   )xr   r   r   rF   i  s    z$itermultiaggregate.<locals>.<lambda>c                       g | ]}  |qS r   indexrA   f)hdrr   r   
<listcomp>}      z&itermultiaggregate.<locals>.<listcomp>c                 3   s    | ]} |V  qd S r(   r   rA   row)	valgetterr   r   rC         z%itermultiaggregate.<locals>.<genexpr>c                 3   s    | ]}|  V  qd S r(   r   rZ   )idxr   r   rC     r]   )r   itemsiterr   	itertoolschainr8   r:   r   r;   rG   r
   r3   appendr   operator
itemgetterrT   )r#   r   r6   itoutfldaggrH   groupedrI   r4   outrowsrcfldaggfunaggvalidxsvalsr   )rW   r^   r\   r   rM   D  sn   






rM   c                 C   s$   t | ||}t|}t||t}|S )z[Group by the `key` field then count the number of distinct values in the
    `value` field.)r   r   r>   rG   )r   r   r5   s1s2s3r   r   r   groupcountdistinctvalues  s   rs   c              	   C      dd }t | ||||||dS )a  Group by the `key` field then return the first row within each group.
    E.g.::

        >>> import petl as etl
        >>> table1 = [['foo', 'bar', 'baz'],
        ...           ['A', 1, True],
        ...           ['C', 7, False],
        ...           ['B', 2, False],
        ...           ['C', 9, True]]
        >>> table2 = etl.groupselectfirst(table1, key='foo')
        >>> table2
        +-----+-----+-------+
        | foo | bar | baz   |
        +=====+=====+=======+
        | 'A' |   1 | True  |
        +-----+-----+-------+
        | 'B' |   2 | False |
        +-----+-----+-------+
        | 'C' |   7 | False |
        +-----+-----+-------+

    See also :func:`petl.transform.reductions.groupselectlast`,
    :func:`petl.transform.dedup.distinct`.

    c                 S   s   t |S r(   )r   )rI   r4   r   r   r   _reducer  s   z"groupselectfirst.<locals>._reducerr   r   r   r   r   r    r   r   r   r   r   r   ru   r   r   r   groupselectfirst  s   
ry   c              	   C   rt   )a#  Group by the `key` field then return the last row within each group.
    E.g.::

        >>> import petl as etl
        >>> table1 = [['foo', 'bar', 'baz'],
        ...           ['A', 1, True],
        ...           ['C', 7, False],
        ...           ['B', 2, False],
        ...           ['C', 9, True]]
        >>> table2 = etl.groupselectlast(table1, key='foo')
        >>> table2
        +-----+-----+-------+
        | foo | bar | baz   |
        +=====+=====+=======+
        | 'A' |   1 | True  |
        +-----+-----+-------+
        | 'B' |   2 | False |
        +-----+-----+-------+
        | 'C' |   9 | True  |
        +-----+-----+-------+

    See also :func:`petl.transform.reductions.groupselectfirst`,
    :func:`petl.transform.dedup.distinct`.

    .. versionadded:: 1.1.0

    c                 S   s   d }|D ]}q|S r(   r   )rI   r4   r[   r   r   r   ru     s   z!groupselectlast.<locals>._reducerrv   rw   rx   r   r   r   groupselectlast  s   
rz   c                 C      t t| |dd|||||dS )zGroup by the `key` field then return the row with the minimum of the
    `value` field within each group. N.B., will only return one row for each
    group, even if multiple rows have the same (minimum) value.Freverser   r   r   r   ry   r   r   r   r5   r   r   r   r   r   r   r   groupselectmin     r   c                 C   r{   )zGroup by the `key` field then return the row with the maximum of the
    `value` field within each group. N.B., will only return one row for each
    group, even if multiple rows have the same (maximum) value.Tr|   r~   r   r   r   r   r   groupselectmax  r   r   c              	   C   s   t | ||||||dS )a  
    Merge duplicate rows under the given key. E.g.::

        >>> import petl as etl
        >>> table1 = [['foo', 'bar', 'baz'],
        ...           ['A', 1, 2.7],
        ...           ['B', 2, None],
        ...           ['D', 3, 9.4],
        ...           ['B', None, 7.8],
        ...           ['E', None, 42.],
        ...           ['D', 3, 12.3],
        ...           ['A', 2, None]]
        >>> table2 = etl.mergeduplicates(table1, 'foo')
        >>> table2
        +-----+------------------+-----------------------+
        | foo | bar              | baz                   |
        +=====+==================+=======================+
        | 'A' | Conflict({1, 2}) |                   2.7 |
        +-----+------------------+-----------------------+
        | 'B' |                2 |                   7.8 |
        +-----+------------------+-----------------------+
        | 'D' |                3 | Conflict({9.4, 12.3}) |
        +-----+------------------+-----------------------+
        | 'E' | None             |                  42.0 |
        +-----+------------------+-----------------------+

    Missing values are overridden by non-missing values. Conflicting values are
    reported as an instance of the Conflict class (sub-class of frozenset).

    If `presorted` is True, it is assumed that the data are already sorted by
    the given key, and the `buffersize`, `tempdir` and `cache` arguments are
    ignored. Otherwise, the data are sorted, see also the discussion of the
    `buffersize`, `tempdir` and `cache` arguments under the
    :func:`petl.transform.sorts.sort` function.

    See also :func:`petl.transform.dedup.conflicts`.

    )missingr   r   r   r   )MergeDuplicatesView)r   r   r   r   r   r   r   r   r   r   mergeduplicates  s   
)r   c                   @   "   e Zd Z		dddZdd ZdS )	r   NFTc                 C   s0   |r|| _ n
t|||||d| _ || _|| _d S r!   )r   r   r   r   )r$   r   r   r   r   r   r   r   r   r   r   r%   8  s   
zMergeDuplicatesView.__init__c                 C   rL   r(   )itermergeduplicatesr   r   r   r*   r   r   r   r+   B  rN   zMergeDuplicatesView.__iter__r-   r.   r   r   r   r   r   6  s
    

r   c                 #   s    t | }t|\}}ttt| t|tr|g}|hnt|}t|fdd D } fdd|D }|| t	|V  t
||D ]2\}tt|trX|g}	nt|}	fdd|D }
fdd|
D }|	| t	|	V  qGd S )Nc                    s   g | ]}| vr|qS r   r   rU   )keyfldsr   r   rX   R  s    z'itermergeduplicates.<locals>.<listcomp>c                    rR   r   rS   rU   )fldsr   r   rX   S  rY   c                    s$   g | ] t  fd dD qS )c                 3   s0    | ]}t | kr|  kr|  V  qd S r(   )rG   rZ   )ir   r   r   rC   ^  s    z1itermergeduplicates.<locals>.<listcomp>.<genexpr>)set)rA   )rJ   r   )r   r   rX   ^  s    c                    s8   g | ]}t |d kr| nt |dkr nt|qS )r@   r   )rG   popConflict)rA   ro   )r   r   r   rX   a  s    )r`   r   r;   mapr	   r:   r   r   extendr3   r   )r   r   r   rf   rW   rH   valflds
valfldidxsrI   rj   
mergedvals
normedvalsr   )r   rJ   r   r   r   r   F  s6   





r   c                  O   s8   d|v sJ d|d }t | i |}t||dd}|S )a  
    Convenience function to combine multiple tables (via
    :func:`petl.transform.sorts.mergesort`) then combine duplicate rows by
    merging under the given key (via
    :func:`petl.transform.reductions.mergeduplicates`). E.g.::

        >>> import petl as etl
        >>> table1 = [['foo', 'bar', 'baz'],
        ...           [1, 'A', True],
        ...           [2, 'B', None],
        ...           [4, 'C', True]]
        >>> table2 = [['bar', 'baz', 'quux'],
        ...           ['A', True, 42.0],
        ...           ['B', False, 79.3],
        ...           ['C', False, 12.4]]
        >>> table3 = etl.merge(table1, table2, key='bar')
        >>> table3
        +-----+-----+-------------------------+------+
        | bar | foo | baz                     | quux |
        +=====+=====+=========================+======+
        | 'A' |   1 | True                    | 42.0 |
        +-----+-----+-------------------------+------+
        | 'B' |   2 | False                   | 79.3 |
        +-----+-----+-------------------------+------+
        | 'C' |   4 | Conflict({False, True}) | 12.4 |
        +-----+-----+-------------------------+------+

    Keyword arguments are the same as for
    :func:`petl.transform.sorts.mergesort`, except `key` is required.

    r   z"keyword argument "key" is requiredT)r   r   )r   r   )tableskwargsr   t1t2r   r   r   mergei  s
   !r   c                       s   e Zd Z fddZ  ZS )r   c                    s   t t| | |}|S r(   )superr   __new__)clsr_   s	__class__r   r   r     s   zConflict.__new__)r/   r0   r1   r   __classcell__r   r   r   r   r     s    r   c              
   C   r   )a  
    Reduce rows recursively via the Python standard :func:`reduce` function.
    E.g.::

        >>> import petl as etl
        >>> table1 = [['id', 'count'],
        ...           [1, 3],
        ...           [1, 5],
        ...           [2, 4],
        ...           [2, 8]]
        >>> import operator
        >>> table2 = etl.fold(table1, 'id', operator.add, 'count',
        ...                   presorted=True)
        >>> table2
        +-----+-------+
        | key | value |
        +=====+=======+
        |   1 |     8 |
        +-----+-------+
        |   2 |    12 |
        +-----+-------+

    See also :func:`petl.transform.reductions.aggregate`,
    :func:`petl.transform.reductions.rowreduce`.

    )r5   r   r   r   r   )FoldView)r   r   rV   r5   r   r   r   r   r   r   r   fold  s   r   c                   @   r   )	r   NFTc	           	      C   s6   |r|| _ n
t|||||d| _ || _|| _|| _d S r!   )r   r   r   rV   r5   )	r$   r   r   rV   r5   r   r   r   r   r   r   r   r%     r&   zFoldView.__init__c                 C   r'   r(   )iterfoldr   r   rV   r5   r*   r   r   r   r+     r,   zFoldView.__iter__r-   r.   r   r   r   r   r     r2   r   c                 c   s2    dV  t | ||D ]\}}|t||fV  q
d S )N)r   r5   )r   r   )r   r   rV   r5   rI   rJ   r   r   r   r     s
   r   r-   )NNFNNTr5   )FNNT)3
__future__r   r   r   ra   rd   collectionsr   petl.compatr   r   r   r	   petl.errorsr
   petl.util.baser   r   r   r   petl.util.countingr   petl.transform.sortsr   r   petl.transform.basicsr   petl.transform.dedupr   r    r   r)   r>   r9   r?   r=   rM   rs   ry   rz   r   r   r   r   r   r   	frozensetr   r   r   r   r   r   r   r   <module>   sx    
-	
 $G

#
(


.#(
!