o ;Di%@sddlmZmZmZddlZddlZddlZddlZddlm Z ddl m Z ddl m Z ddd Zdddejfd d Zee _ee _Gd d d e ZGdddeZGdddeZddZee _Gddde ZdS))absolute_importprint_functiondivisionN)PY3)Table) onlinestatscCst||||S)a Report progress on rows passing through to a file or file-like object (defaults to sys.stderr). E.g.:: >>> import petl as etl >>> table = etl.dummytable(100000) >>> table.progress(10000).tocsv('example.csv') # doctest: +SKIP 10000 rows in 0.13s (78363 row/s); batch in 0.13s (78363 row/s) 20000 rows in 0.22s (91679 row/s); batch in 0.09s (110448 row/s) 30000 rows in 0.31s (96573 row/s); batch in 0.09s (108114 row/s) 40000 rows in 0.40s (99535 row/s); batch in 0.09s (109625 row/s) 50000 rows in 0.49s (101396 row/s); batch in 0.09s (109591 row/s) 60000 rows in 0.59s (102245 row/s); batch in 0.09s (106709 row/s) 70000 rows in 0.68s (103221 row/s); batch in 0.09s (109498 row/s) 80000 rows in 0.77s (103810 row/s); batch in 0.09s (108126 row/s) 90000 rows in 0.90s (99465 row/s); batch in 0.13s (74516 row/s) 100000 rows in 1.02s (98409 row/s); batch in 0.11s (89821 row/s) 100000 rows in 1.02s (98402 row/s); batches in 0.10 +/- 0.02s [0.09-0.13] (100481 +/- 13340 rows/s [74516-110448]) See also :func:`petl.util.timing.clock`. ) ProgressView)table batchsizeprefixoutrI/var/www/Datamplify/venv/lib/python3.10/site-packages/petl/util/timing.pyprogresssrcCst|||||dS)a Report progress on rows passing through to a python logger. If logger is none, a new logger will be created that, by default, streams to stdout. E.g.:: >>> import petl as etl >>> table = etl.dummytable(100000) >>> table.log_progress(10000).tocsv('example.csv') # doctest: +SKIP 10000 rows in 0.13s (78363 row/s); batch in 0.13s (78363 row/s) 20000 rows in 0.22s (91679 row/s); batch in 0.09s (110448 row/s) 30000 rows in 0.31s (96573 row/s); batch in 0.09s (108114 row/s) 40000 rows in 0.40s (99535 row/s); batch in 0.09s (109625 row/s) 50000 rows in 0.49s (101396 row/s); batch in 0.09s (109591 row/s) 60000 rows in 0.59s (102245 row/s); batch in 0.09s (106709 row/s) 70000 rows in 0.68s (103221 row/s); batch in 0.09s (109498 row/s) 80000 rows in 0.77s (103810 row/s); batch in 0.09s (108126 row/s) 90000 rows in 0.90s (99465 row/s); batch in 0.13s (74516 row/s) 100000 rows in 1.02s (98409 row/s); batch in 0.11s (89821 row/s) 100000 rows in 1.02s (98402 row/s); batches in 0.10 +/- 0.02s [0.09-0.13] (100481 +/- 13340 rows/s [74516-110448]) See also :func:`petl.util.timing.clock`. )level)LoggingProgressView)r r r loggerrrrr log_progress*src@s.eZdZdZddZejddZddZdS) ProgressViewBasezA Abstract base class for reporting on proecessing status cCs||_||_||_dSN)innerr r )selfrr r rrr__init__Ns zProgressViewBase.__init__cCsdSrrrmessagerrr print_messageSszProgressViewBase.print_messagec cs\t}|}d}d\}}d\}}d\}} t|jD]\} } | |jdkr| dkr|d7}t} | |} |dus=| |kr?| }|dusG| |krI| }| |}zt| |}Wn ty`d}Ynwz t|j| }Wn tyud}Ynw| ||| |f}|jd|}||| }t| |||d\}}t|||| d\}} | Vqt}||}zt| |}Wn tyd}Ynw|dkr|durd}|durd}z t|j|}Wn tyd}Ynwz t|j|}Wn tyd}Ynw| ||||d||t|t| dt|t|f }|jd|}n | ||f}|jd |}||dS) Nr)NN)rrz6%s rows in %.2fs (%s row/s); batch in %.2fs (%s row/s))meanvarianceg?z]%s rows in %.2fs (%s row/s); batches in %.2f +/- %.2fs [%.2f-%.2f] (%s +/- %s rows/s [%s-%s])z%s rows in %.2fs (%s row/s)) time enumeraterr intZeroDivisionErrorr rr)rstart batchstartbatchn batchtimemin batchtimemax batchtimemean batchtimevar batchratemean batchratevarnrbatchend batchtime elapsedtimerate batchratevrend batchratemin batchratemaxrrr__iter__Ws         zProgressViewBase.__iter__N) __name__ __module__ __qualname____doc__rabcabstractmethodrr9rrrrrIs   rcs(eZdZdZfddZddZZS)r zM Reports progress to a file_object like sys.stdout or a file handler cs0|dur tj|_n||_tt||||dSr)sysstderr file_objectsuperr r)rrr r r __class__rrrs zProgressView.__init__cCs,t||jdt|jdr|jdSdS)N)fileflush)printrBhasattrrGrrrrrs zProgressView.print_message)r:r;r<r=rr __classcell__rrrDrr s r cs.eZdZdZejffdd ZddZZS)rzC Reports progress to a logger, log handler, or log adapter csF|durtt|_|j|n||_||_tt||||dSr) logging getLoggerr:rsetLevelrrCrr)rrr r rrrDrrrs  zLoggingProgressView.__init__cCs|j|j|dSr)rlogrrrrrrsz!LoggingProgressView.print_message) r:r;r<r=rKINFOrrrJrrrDrrs rcCst|S)a Time how long is spent retrieving rows from the wrapped container. Enables diagnosis of which steps in a pipeline are taking the most time. E.g.:: >>> import petl as etl >>> t1 = etl.dummytable(100000) >>> c1 = etl.clock(t1) >>> t2 = etl.convert(c1, 'foo', lambda v: v**2) >>> c2 = etl.clock(t2) >>> p = etl.progress(c2, 10000) >>> etl.tocsv(p, 'example.csv') # doctest: +SKIP 10000 rows in 0.23s (44036 row/s); batch in 0.23s (44036 row/s) 20000 rows in 0.38s (52167 row/s); batch in 0.16s (63979 row/s) 30000 rows in 0.54s (55749 row/s); batch in 0.15s (64624 row/s) 40000 rows in 0.69s (57765 row/s); batch in 0.15s (64793 row/s) 50000 rows in 0.85s (59031 row/s); batch in 0.15s (64707 row/s) 60000 rows in 1.00s (59927 row/s); batch in 0.15s (64847 row/s) 70000 rows in 1.16s (60483 row/s); batch in 0.16s (64051 row/s) 80000 rows in 1.31s (61008 row/s); batch in 0.15s (64953 row/s) 90000 rows in 1.47s (61356 row/s); batch in 0.16s (64285 row/s) 100000 rows in 1.62s (61703 row/s); batch in 0.15s (65012 row/s) 100000 rows in 1.62s (61700 row/s); batches in 0.16 +/- 0.02s [0.15-0.23] (62528 +/- 6173 rows/s [44036-65012]) >>> # time consumed retrieving rows from t1 ... c1.time # doctest: +SKIP 0.7243089999999492 >>> # time consumed retrieving rows from t2 ... c2.time # doctest: +SKIP 1.1704209999999766 >>> # actual time consumed by the convert step ... c2.time - c1.time # doctest: +SKIP 0.4461120000000274 See also :func:`petl.util.timing.progress`. ) ClockView)r rrrclocks%rQc@seZdZddZddZdS)rPcCs ||_dSr)wrapped)rrRrrrrs zClockView.__init__ccsxd|_t|j} trtnt}zt|}Wn ty$YdSwtr+tnt}|j||7_|Vq )Nr)r!iterrRr perf_counterrQnext StopIteration)ritbeforerowafterrrrr9s   zClockView.__iter__N)r:r;r<rr9rrrrrPs rP)rr N) __future__rrrr>rKr@r! petl.compatrpetl.util.baserpetl.util.statisticsrrrOrrr rrQrPrrrrs$    Y(