o ;Di@sVddlmZmZmZddlmZmZddlmZddl m Z m Z m Z m Z mZddlmZddlmZddlmZdJd d ZdJd dZdKddZdKddZ dLddZee _ddZGdddeZ dMddZee _GdddeZdNdd Zee _ dOd!d"Z e e _ dLd#d$Z!e!e _! dMd%d&Z"e"e _" dNd'd(Z#e#e _#  dPd)d*Z$e$e _$   dQd+d,Z%e%e _%Gd-d.d.e Z&   dRd/d0Z'e'e _'Gd1d2d2e Z(   dSd3d4Z)e)e _)Gd5d6d6e Z* dTd7d8Z+ dUd9d:Z,e,e _,  dUd;d<Z-e-e _-Gd=d>d>e Z.d?d@Z/ddAl0m1Z1e1dBdCZ2dJdDdEZ3e3e _3dFdGZ4dHdIZ5d S)V)absolute_importprint_functiondivision) itemgetter attrgetter) text_type) asindicesrecordsTablevalues rowgroupby)DuplicateKeyError)addfield)sortstartstopNcCsddl}|}t|}t|}ttt|}||vsJd||vs'Jdt||} t||} |dur>> import petl as etl >>> table = [['start', 'stop', 'value'], ... [1, 4, 'foo'], ... [3, 7, 'bar'], ... [4, 9, 'baz']] >>> lkp = etl.intervallookup(table, 'start', 'stop') >>> lkp.search(0, 1) [] >>> lkp.search(1, 2) [(1, 4, 'foo')] >>> lkp.search(2, 4) [(1, 4, 'foo'), (3, 7, 'bar')] >>> lkp.search(2, 5) [(1, 4, 'foo'), (3, 7, 'bar'), (4, 9, 'baz')] >>> lkp.search(9, 14) [] >>> lkp.search(19, 140) [] >>> lkp.search(0) [] >>> lkp.search(1) [(1, 4, 'foo')] >>> lkp.search(2) [(1, 4, 'foo')] >>> lkp.search(4) [(3, 7, 'bar'), (4, 9, 'baz')] >>> lkp.search(5) [(3, 7, 'bar'), (4, 9, 'baz')] Note start coordinates are included and stop coordinates are excluded from the interval. Use the `include_stop` keyword argument to include the upper bound of the interval when finding overlaps. Some examples using the `include_stop` and `value` keyword arguments:: >>> import petl as etl >>> table = [['start', 'stop', 'value'], ... [1, 4, 'foo'], ... [3, 7, 'bar'], ... [4, 9, 'baz']] >>> lkp = etl.intervallookup(table, 'start', 'stop', include_stop=True, ... value='value') >>> lkp.search(0, 1) ['foo'] >>> lkp.search(1, 2) ['foo'] >>> lkp.search(2, 4) ['foo', 'bar', 'baz'] >>> lkp.search(2, 5) ['foo', 'bar', 'baz'] >>> lkp.search(9, 14) ['baz'] >>> lkp.search(19, 140) [] >>> lkp.search(0) [] >>> lkp.search(1) ['foo'] >>> lkp.search(2) ['foo'] >>> lkp.search(4) ['foo', 'bar', 'baz'] >>> lkp.search(5) ['bar', 'baz'] rrr  include_stop)r,IntervalTreeLookup)rrrr r9r!r*r*r+intervallookuposH r;cCsz|dur|r|d}|d8}||f}n|f}n|r!|d7}|d8}||f}t|dkr4t|j|}|St|j|}|S)N)rsortedoverlapat)r!rrr9argsresultsr*r*r+ _search_trees  rCc@s$eZdZdddZdddZeZdS) r:FcCs||_||_dSN)r!r9)selfr!r9r*r*r+__init__s zIntervalTreeLookup.__init__NcCs t|j|||j}dd|DS)NcSsg|]}|jqSr*)data).0rr*r*r+ sz-IntervalTreeLookup.search..)rCr!r9rErrrBr*r*r+searchszIntervalTreeLookup.searchFrD__name__ __module__ __qualname__rFrLfindr*r*r*r+r:s  r:TcCst||||d}t|||dS)a Construct an interval lookup for the given table, returning at most one result for each query. E.g.:: >>> import petl as etl >>> table = [['start', 'stop', 'value'], ... [1, 4, 'foo'], ... [3, 7, 'bar'], ... [4, 9, 'baz']] >>> lkp = etl.intervallookupone(table, 'start', 'stop', strict=False) >>> lkp.search(0, 1) >>> lkp.search(1, 2) (1, 4, 'foo') >>> lkp.search(2, 4) (1, 4, 'foo') >>> lkp.search(2, 5) (1, 4, 'foo') >>> lkp.search(9, 14) >>> lkp.search(19, 140) >>> lkp.search(0) >>> lkp.search(1) (1, 4, 'foo') >>> lkp.search(2) (1, 4, 'foo') >>> lkp.search(4) (3, 7, 'bar') >>> lkp.search(5) (3, 7, 'bar') If ``strict=True``, queries returning more than one result will raise a `DuplicateKeyError`. If ``strict=False`` and there is more than one result, the first result is returned. Note start coordinates are included and stop coordinates are excluded from the interval. Use the `include_stop` keyword argument to include the upper bound of the interval when finding overlaps. r7)strictr9)r,IntervalTreeLookupOne)rrrr r9rSr!r*r*r+intervallookupones)rUc@s$eZdZdddZd ddZeZdS) rTTFcCs||_||_||_dSrD)r!rSr9)rEr!rSr9r*r*r+rFs zIntervalTreeLookupOne.__init__NcCsJt|j|||j}t|dkrdSt|dkr |jr t||f|djS)Nrr<)rCr!r9rrSr rGrKr*r*r+rLs    zIntervalTreeLookupOne.search)TFrDrNr*r*r*r+rTs   rTcCst|||d}t||dS)zg As :func:`petl.transform.intervals.intervallookup` but return records instead of tuples. rrr8)r5r:)rrrr9r!r*r*r+intervalrecordlookup"s rWcCst|||d}t|||dS)zj As :func:`petl.transform.intervals.intervallookupone` but return records instead of tuples. rVr9rS)r5rT)rrrr9rSr!r*r*r+intervalrecordlookupone0srYc Cs:t|||||d}t}|D] }t|||d||<q|S)a Construct a faceted interval lookup for the given table. E.g.:: >>> import petl as etl >>> table = (('type', 'start', 'stop', 'value'), ... ('apple', 1, 4, 'foo'), ... ('apple', 3, 7, 'bar'), ... ('orange', 4, 9, 'baz')) >>> lkp = etl.facetintervallookup(table, key='type', start='start', stop='stop') >>> lkp['apple'].search(1, 2) [('apple', 1, 4, 'foo')] >>> lkp['apple'].search(2, 4) [('apple', 1, 4, 'foo'), ('apple', 3, 7, 'bar')] >>> lkp['apple'].search(2, 5) [('apple', 1, 4, 'foo'), ('apple', 3, 7, 'bar')] >>> lkp['orange'].search(2, 5) [('orange', 4, 9, 'baz')] >>> lkp['orange'].search(9, 14) [] >>> lkp['orange'].search(19, 140) [] >>> lkp['apple'].search(1) [('apple', 1, 4, 'foo')] >>> lkp['apple'].search(2) [('apple', 1, 4, 'foo')] >>> lkp['apple'].search(4) [('apple', 3, 7, 'bar')] >>> lkp['apple'].search(5) [('apple', 3, 7, 'bar')] >>> lkp['orange'].search(5) [('orange', 4, 9, 'baz')] r7r8)r3r-r:) rr.rrr r9r1outr2r*r*r+facetintervallookup?s %r[c Cs<t|||||d}t}|D] } t|| ||d|| <q|S)a7 Construct a faceted interval lookup for the given table, returning at most one result for each query. If ``strict=True``, queries returning more than one result will raise a `DuplicateKeyError`. If ``strict=False`` and there is more than one result, the first result is returned. r7rX)r3r-rT) rr.rrr r9rSr1rZr2r*r*r+facetintervallookuponens   r\cCs8t||||d}t}|D] }t|||d||<q |S)zZ As :func:`petl.transform.intervals.facetintervallookup` but return records. rVr8)r6r-r:)rr.rrr9r1rZr2r*r*r+facetintervalrecordlookups r]c Cs:t||||d}t}|D] }t||||d||<q |S)z] As :func:`petl.transform.intervals.facetintervallookupone` but return records. rVrX)r6r-rT) rr.rrr9rSr1rZr2r*r*r+facetintervalrecordlookupones  r^c Cs6|du|duks Jdt|||||||||| | d S)a Join two tables by overlapping intervals. E.g.:: >>> import petl as etl >>> left = [['begin', 'end', 'quux'], ... [1, 2, 'a'], ... [2, 4, 'b'], ... [2, 5, 'c'], ... [9, 14, 'd'], ... [1, 1, 'e'], ... [10, 10, 'f']] >>> right = [['start', 'stop', 'value'], ... [1, 4, 'foo'], ... [3, 7, 'bar'], ... [4, 9, 'baz']] >>> table1 = etl.intervaljoin(left, right, ... lstart='begin', lstop='end', ... rstart='start', rstop='stop') >>> table1.lookall() +-------+-----+------+-------+------+-------+ | begin | end | quux | start | stop | value | +=======+=====+======+=======+======+=======+ | 1 | 2 | 'a' | 1 | 4 | 'foo' | +-------+-----+------+-------+------+-------+ | 2 | 4 | 'b' | 1 | 4 | 'foo' | +-------+-----+------+-------+------+-------+ | 2 | 4 | 'b' | 3 | 7 | 'bar' | +-------+-----+------+-------+------+-------+ | 2 | 5 | 'c' | 1 | 4 | 'foo' | +-------+-----+------+-------+------+-------+ | 2 | 5 | 'c' | 3 | 7 | 'bar' | +-------+-----+------+-------+------+-------+ | 2 | 5 | 'c' | 4 | 9 | 'baz' | +-------+-----+------+-------+------+-------+ >>> # include stop coordinate in intervals ... table2 = etl.intervaljoin(left, right, ... lstart='begin', lstop='end', ... rstart='start', rstop='stop', ... include_stop=True) >>> table2.lookall() +-------+-----+------+-------+------+-------+ | begin | end | quux | start | stop | value | +=======+=====+======+=======+======+=======+ | 1 | 2 | 'a' | 1 | 4 | 'foo' | +-------+-----+------+-------+------+-------+ | 2 | 4 | 'b' | 1 | 4 | 'foo' | +-------+-----+------+-------+------+-------+ | 2 | 4 | 'b' | 3 | 7 | 'bar' | +-------+-----+------+-------+------+-------+ | 2 | 4 | 'b' | 4 | 9 | 'baz' | +-------+-----+------+-------+------+-------+ | 2 | 5 | 'c' | 1 | 4 | 'foo' | +-------+-----+------+-------+------+-------+ | 2 | 5 | 'c' | 3 | 7 | 'bar' | +-------+-----+------+-------+------+-------+ | 2 | 5 | 'c' | 4 | 9 | 'baz' | +-------+-----+------+-------+------+-------+ | 9 | 14 | 'd' | 4 | 9 | 'baz' | +-------+-----+------+-------+------+-------+ | 1 | 1 | 'e' | 1 | 4 | 'foo' | +-------+-----+------+-------+------+-------+ Note start coordinates are included and stop coordinates are excluded from the interval. Use the `include_stop` keyword argument to include the upper bound of the interval when finding overlaps. An additional key comparison can be made, e.g.:: >>> import petl as etl >>> left = (('fruit', 'begin', 'end'), ... ('apple', 1, 2), ... ('apple', 2, 4), ... ('apple', 2, 5), ... ('orange', 2, 5), ... ('orange', 9, 14), ... ('orange', 19, 140), ... ('apple', 1, 1)) >>> right = (('type', 'start', 'stop', 'value'), ... ('apple', 1, 4, 'foo'), ... ('apple', 3, 7, 'bar'), ... ('orange', 4, 9, 'baz')) >>> table3 = etl.intervaljoin(left, right, ... lstart='begin', lstop='end', lkey='fruit', ... rstart='start', rstop='stop', rkey='type') >>> table3.lookall() +----------+-------+-----+----------+-------+------+-------+ | fruit | begin | end | type | start | stop | value | +==========+=======+=====+==========+=======+======+=======+ | 'apple' | 1 | 2 | 'apple' | 1 | 4 | 'foo' | +----------+-------+-----+----------+-------+------+-------+ | 'apple' | 2 | 4 | 'apple' | 1 | 4 | 'foo' | +----------+-------+-----+----------+-------+------+-------+ | 'apple' | 2 | 4 | 'apple' | 3 | 7 | 'bar' | +----------+-------+-----+----------+-------+------+-------+ | 'apple' | 2 | 5 | 'apple' | 1 | 4 | 'foo' | +----------+-------+-----+----------+-------+------+-------+ | 'apple' | 2 | 5 | 'apple' | 3 | 7 | 'bar' | +----------+-------+-----+----------+-------+------+-------+ | 'orange' | 2 | 5 | 'orange' | 4 | 9 | 'baz' | +----------+-------+-----+----------+-------+------+-------+ N:facet key field must be provided for both or neither table) lstartlstoprstartrstoplkeyrkeyr9lprefixrprefix)IntervalJoinView) leftrightr`rarbrcrdrer9rfrgr*r*r+ intervaljoinsk rkc@$eZdZ   d ddZddZdS) rhrrNFc CsF||_||_||_||_||_||_||_||_| |_| |_ | |_ dSrD) rir`rardrjrbrcrer9rfrg) rErirjr`rarbrcrdrer9rfrgr*r*r+rF!s zIntervalJoinView.__init__cCs8t|j|j|j|j|j|j|j|j|j d|j |j dd S)NF rirjr`rarbrcrdrer9missingrfrg leftouter) iterintervaljoinrirjr`rarbrcrdrer9rfrgrEr*r*r+__iter__0szIntervalJoinView.__iter__ rrrrNNFNNrOrPrQrFrrr*r*r*r+rhs  rhc Cs8|du|duks Jdt|||||||||| | | d S)aq Like :func:`petl.transform.intervals.intervaljoin` but rows from the left table without a match in the right table are also included. E.g.:: >>> import petl as etl >>> left = [['begin', 'end', 'quux'], ... [1, 2, 'a'], ... [2, 4, 'b'], ... [2, 5, 'c'], ... [9, 14, 'd'], ... [1, 1, 'e'], ... [10, 10, 'f']] >>> right = [['start', 'stop', 'value'], ... [1, 4, 'foo'], ... [3, 7, 'bar'], ... [4, 9, 'baz']] >>> table1 = etl.intervalleftjoin(left, right, ... lstart='begin', lstop='end', ... rstart='start', rstop='stop') >>> table1.lookall() +-------+-----+------+-------+------+-------+ | begin | end | quux | start | stop | value | +=======+=====+======+=======+======+=======+ | 1 | 2 | 'a' | 1 | 4 | 'foo' | +-------+-----+------+-------+------+-------+ | 2 | 4 | 'b' | 1 | 4 | 'foo' | +-------+-----+------+-------+------+-------+ | 2 | 4 | 'b' | 3 | 7 | 'bar' | +-------+-----+------+-------+------+-------+ | 2 | 5 | 'c' | 1 | 4 | 'foo' | +-------+-----+------+-------+------+-------+ | 2 | 5 | 'c' | 3 | 7 | 'bar' | +-------+-----+------+-------+------+-------+ | 2 | 5 | 'c' | 4 | 9 | 'baz' | +-------+-----+------+-------+------+-------+ | 9 | 14 | 'd' | None | None | None | +-------+-----+------+-------+------+-------+ | 1 | 1 | 'e' | None | None | None | +-------+-----+------+-------+------+-------+ | 10 | 10 | 'f' | None | None | None | +-------+-----+------+-------+------+-------+ Note start coordinates are included and stop coordinates are excluded from the interval. Use the `include_stop` keyword argument to include the upper bound of the interval when finding overlaps. Nr_) r`rarbrcrdrer9rnrfrg)IntervalLeftJoinView) rirjr`rarbrcrdrer9rnrfrgr*r*r+intervalleftjoinBs3 rvc@rl) rurrNFc CsL||_||_||_||_||_||_||_||_| |_| |_ | |_ | |_ dSrD) rir`rardrjrbrcrernr9rfrg) rErirjr`rarbrcrdrernr9rfrgr*r*r+rFs zIntervalLeftJoinView.__init__cCs:t|j|j|j|j|j|j|j|j|j |j |j |j dd S)NTrm) rprirjr`rarbrcrdrer9rnrfrgrqr*r*r+rrszIntervalLeftJoinView.__iter__) rrrrNNNFNNrtr*r*r*r+rus  ruc Cs4|du|duks Jdt|||||||||| d S)a1 Return rows from the `left` table with no overlapping rows from the `right` table. Note start coordinates are included and stop coordinates are excluded from the interval. Use the `include_stop` keyword argument to include the upper bound of the interval when finding overlaps. Nr_)r`rarbrcrdrer9rn)IntervalAntiJoinView) rirjr`rarbrcrdrer9rnr*r*r+intervalantijoins  rxc@$eZdZ   d ddZddZdS) rwrrNFc Cs@||_||_||_||_||_||_||_||_| |_| |_ dSrD) rir`rardrjrbrcrernr9) rErirjr`rarbrcrdrernr9r*r*r+rFs zIntervalAntiJoinView.__init__cCs8t|j|j|j|j|j|j|j|j|j |j dddddS)NT)rirjr`rarbrcrdrer9rnrfrgroanti) rprirjr`rarbrcrdrer9rnrqr*r*r+rrs zIntervalAntiJoinView.__iter__)rrrrNNNFrtr*r*r*r+rws  rwc! #st|}t|}ttt|}t|}t|}ttt|}t||t|||dur2t||t||t|||durEt||durUt|}| sT||ntfdd|D}| sn|fdd|Dt|Vt| |}t| |}|durt ||||d}|j }|D]>}||}||}|||}|r| s|D]}t|}||t|Vqq| rt|}| s|| gt |t|VqdSt |||||d}t}|D] }||j ||<qtt||} |D]c}| |}||}||}z ||||}Wntyd}Yn ty"d}Ynw|r>| s=|D]}t|}||t|Vq+q| rWt|}| sR|| gt |t|VqdS)Nc3|]}|VqdSrDr*rHf)rfr*r+ z#iterintervaljoin..c3r{rDr*r|)rgr*r+r~rr8r.rrr9)rrrrrrextendrrrr;rLrr[r-KeyErrorAttributeError)!rirjr`rarbrcrdrer9rnrfrgrorzlitlhdrlfldsritrhdrrfldsouthdr getlstartgetlstoplookuprLlrowrrrrowsrrowoutrowr}getlkeyr*)rfrgr+rps              rpc svdu|duks Jddur"t||||| dfdd} nt|||||| dfdd} t||| S)aJ Convenience function to join the left table with values from a specific field in the right hand table. Note start coordinates are included and stop coordinates are excluded from the interval. Use the `include_stop` keyword argument to include the upper bound of the interval when finding overlaps. Nr_)rrr r9cs||SrDrLr))lkpr`rar*r+Pz$intervaljoinvalues..cs|||SrDrrrdrr`rar*r+rTs)r;r[r) rirjr r`rarbrcrdrer9r}r*rr+intervaljoinvalues>s    rc Cs2|du|duks Jdt|||||||||d S)ze Subtract intervals in the right hand table from intervals in the left hand table. Nr_)r`rarbrcrdrer9)IntervalSubtractView) rirjr`rarbrcrdrer9r*r*r+intervalsubtract[s rc@ry) rrrNFc Cs:||_||_||_||_||_||_||_||_| |_dSrD) rir`rardrjrbrcrer9) rErirjr`rarbrcrdrer9r*r*r+rFos zIntervalSubtractView.__init__c Cs*t|j|j|j|j|j|j|j|j|j SrD) iterintervalsubtractrirjr`rarbrcrdrer9rqr*r*r+rr|s  zIntervalSubtractView.__iter__rrrrNNFrtr*r*r*r+rms  rc  #sNt|} t| } ttt| } t|} t| } t| |t| ||dur+t| |t| |t| ||dur>t| |t| }t|Vt| ||f\}}t||}tt| ||f|durt||||d}|j }| D]@}||\}}|||}|st|Vqot fdd|Dtdd}t |||D]\}}t|}|||<|||<t|VqqodSt |||||d}tt| |}| D]`}||}||\}}z || ||}Wnt yd}Yn tyd}Ynw|st|Vqt fdd|Dtdd}t |||D]\}}t|}|||<|||<t|Vq qdS)Nr8cg|]}|qSr*r*rHr getrcoordsr*r+rJrz(iterintervalsubtract..rr.rcrr*r*rrr*r+rJr)rrrrrrrrr;rLr> _subtractr[rr)rirjr`rarbrcrdrer9rrrrrr lstartidxlstopidx getlcoordsrrLrrrrrivsxyrZrr*rr+rs~                 r) namedtupleIntervalz start stopccs|durt||d}tt|||fD]}|VqdSt|||fd}t||||fdD]\}}t|D]}|f|Vq5q-dS)z Utility function to collapse intervals in a table. If no facet `key` is given, returns an iterator over `(start, stop)` tuples. If facet `key` is given, returns an iterator over `(key, start, stop)` tuples. Nr)r.r )r _collapser r )rrrr.ivr2gr*r*r+collapsedintervalss  rccsd}|D]/\}}|durt||}q||jkr|kr'nnt|j|}q||jkr4|Vt||}q|dur>|VdSdS)zG Collapse an iterable of intervals sorted by start coord. N) _Intervalrr) intervalsspanrrr*r*r+rs     rccs\|}d}t|D]\}}||krt||V|}q |dur*||kr,t||VdSdSdS)z; Subtract intervals from a spanning interval. N)rr)rrrremainder_startsub_stop sub_startr*r*r+rs r)rrNrV)rrNF)rrNFT)rrF)rrFT)FTrs) rrrrNNFNNN)rrrrNNFNrMr)6 __future__rrroperatorrr petl.compatrpetl.util.baserr r r r petl.errorsr petl.transform.basicsrpetl.transform.sortsrr,r3r5r6r;rCobjectr:rUrTrWrYr[r\r]r^rkrhrvrurxrwrprrrr collectionsrrrrrr*r*r*r+s       "  L  -     ,    s# <$ % ^   M