o ;Di"5@sddlmZmZmZddlZddlZddlZddlmZddl m Z ddl m Z ddl mZddl mZddlmZmZdd lmZmZmZmZd d ZGd d d eZd"ddZGdddeZGdddeZddZddZd#ddZ e e_  d$ddZ!e!e_!ddZ"d%d d!Z#dS)&)absolute_importprint_functiondivisionN) JSONEncoder)unlink)NamedTemporaryFile)PY2)pickle)read_source_from_argwrite_source_from_arg)dataTabledictsiterpeekcOst|}t|g|Ri|S)aB Extract data from a JSON file. The file must contain a JSON array as the top level object, and each member of the array will be treated as a row of data. E.g.:: >>> import petl as etl >>> data = ''' ... [{"foo": "a", "bar": 1}, ... {"foo": "b", "bar": 2}, ... {"foo": "c", "bar": 2}] ... ''' >>> with open('example.file1.json', 'w') as f: ... f.write(data) ... 74 >>> table1 = etl.fromjson('example.file1.json', header=['foo', 'bar']) >>> table1 +-----+-----+ | foo | bar | +=====+=====+ | 'a' | 1 | +-----+-----+ | 'b' | 2 | +-----+-----+ | 'c' | 2 | +-----+-----+ Setting argument `lines` to `True` will enable to infer the document as a JSON lines document. For more details about JSON lines please visit https://jsonlines.org/. >>> import petl as etl >>> data_with_jlines = '''{"name": "Gilbert", "wins": [["straight", "7S"], ["one pair", "10H"]]} ... {"name": "Alexa", "wins": [["two pair", "4S"], ["two pair", "9S"]]} ... {"name": "May", "wins": []} ... {"name": "Deloise", "wins": [["three of a kind", "5S"]]}''' ... >>> with open('example.file2.json', 'w') as f: ... f.write(data_with_jlines) ... 223 >>> table2 = etl.fromjson('example.file2.json', lines=True) >>> table2 +-----------+-------------------------------------------+ | name | wins | +===========+===========================================+ | 'Gilbert' | [['straight', '7S'], ['one pair', '10H']] | +-----------+-------------------------------------------+ | 'Alexa' | [['two pair', '4S'], ['two pair', '9S']] | +-----------+-------------------------------------------+ | 'May' | [] | +-----------+-------------------------------------------+ | 'Deloise' | [['three of a kind', '5S']] | +-----------+-------------------------------------------+ If your JSON file does not fit this structure, you will need to parse it via :func:`json.load` and select the array to treat as the data, see also :func:`petl.io.json.fromdicts`. .. versionchanged:: 1.1.0 If no `header` is specified, fields will be discovered by sampling keys from the first `sample` objects in `source`. The header will be constructed from keys in the order discovered. Note that this ordering may not be stable, and therefore it may be advisable to specify an explicit `header` or to use another function like :func:`petl.transform.headers.sortheader` on the resulting table to guarantee stability. )r JsonView)sourceargskwargsrE/var/www/Datamplify/venv/lib/python3.10/site-packages/petl/io/json.pyfromjsonsHrc@seZdZddZddZdS)rcOsN||_|dd|_|dd|_|dd|_|dd|_||_||_dS)NmissingheadersamplelinesF)rpoprrrrrr)selfrrrrrr__init__`s zJsonView.__init__c cs|jdU}tstj|dddd}z8|jr&t||j|jD]}|Vqnt j |g|j Ri|j }t ||j|j|jD]}|Vq>WtsK|ntsS|wwWddS1s_wYdS)Nrbutf-8Tencodingnewline write_through)ropenrio TextIOWrapperr iterjlinesrrjsonloadrr iterdictsrdetach)rfrowrrrr__iter__is2   "zJsonView.__iter__N__name__ __module__ __qualname__rr0rrrrr_s rrcCs"t|rtnt}|||||dS)a8 View a sequence of Python :class:`dict` as a table. E.g.:: >>> import petl as etl >>> dicts = [{"foo": "a", "bar": 1}, ... {"foo": "b", "bar": 2}, ... {"foo": "c", "bar": 2}] >>> table1 = etl.fromdicts(dicts, header=['foo', 'bar']) >>> table1 +-----+-----+ | foo | bar | +=====+=====+ | 'a' | 1 | +-----+-----+ | 'b' | 2 | +-----+-----+ | 'c' | 2 | +-----+-----+ Argument `dicts` can also be a generator, the output of generator is iterated and cached using a temporary file to support further transforms and multiple passes of the table: >>> import petl as etl >>> dicts = ({"foo": chr(ord("a")+i), "bar":i+1} for i in range(3)) >>> table1 = etl.fromdicts(dicts, header=['foo', 'bar']) >>> table1 +-----+-----+ | foo | bar | +=====+=====+ | 'a' | 1 | +-----+-----+ | 'b' | 2 | +-----+-----+ | 'c' | 3 | +-----+-----+ If `header` is not specified, `sample` items from `dicts` will be inspected to discovery dictionary keys. Note that the order in which dictionary keys are discovered may not be stable, See also :func:`petl.io.json.fromjson`. .. versionchanged:: 1.1.0 If no `header` is specified, fields will be discovered by sampling keys from the first `sample` dictionaries in `dicts`. The header will be constructed from keys in the order discovered. Note that this ordering may not be stable, and therefore it may be advisable to specify an explicit `header` or to use another function like :func:`petl.transform.headers.sortheader` on the resulting table to guarantee stability. .. versionchanged:: 1.7.5 Full support of generators passed as `dicts` has been added, leveraging `itertools.tee`. .. versionchanged:: 1.7.11 Generator support has been modified to use temporary file cache instead of `itertools.tee` due to high memory usage. )rrr)inspect isgeneratorDictsGeneratorView DictsView)rrrrviewrrr fromdicts}sAr:c@seZdZdddZddZdS)r8NrcCs||_||_||_||_dSN)r_headerrrrrrrrrrrrs zDictsView.__init__cCst|j|j|j|jSr;)r,rr<rrrrrrr0szDictsView.__iter__NrNr1rrrrr8s  r8cs6eZdZd fdd ZddZddZd d ZZS) r7Nrcs&tt|||||d|_d|_dS)Nr)superr7r _filecache_cachedr= __class__rrrs zDictsGeneratorView.__init__c#sjsjVjs"trtdddd_ntdddd_d}tj} |jkrDj|t j}j }|Vq)zt |Wn t yTYdSwtfddjD}jjt j|jd d j _}|Vq*) NFzwb+r)deletemodebufsize)rErF bufferingTc3s|] }|jVqdSr;)getr.0r.orrr sz.DictsGeneratorView.__iter__..)protocol)r<_determine_headerrArriterrrBseekr r+tellnext StopIterationtupledump)rpositionitr/rrLrr0s8       zDictsGeneratorView.__iter__csrt|j}tt||j\}}||_t|tr|g}|D]}t|dr1fdd|D7qt |_ |S)Nkeyscg|]}|vr|qSrrrKkrrr z8DictsGeneratorView._determine_header..) rRrlistrr isinstancedicthasattrr[rWr<)rrZpeekrMrr_rrQs    z$DictsGeneratorView._determine_headercCs$|jr|jt|jjdSdSr;)rAclosernamer>rrr__del__s zDictsGeneratorView.__del__r?)r2r3r4rr0rQri __classcell__rrrCrr7s  r7c#st|}dur*tt|d\}}t|tdr*fddD7tV|D]}t|tfddDVq1dS)Nr[cr\rrr]r_rrr`raziterjlines..c3s$|] }|vr |nVqdSr;rrJ)json_objrrrrNs"ziterjlines..)rRrbrr*loadsrer[rW)r.rrrZrfrMr)rrlrrr)s    r)c#st|}dur*tt||\}}|D]tdr)fddD7qtV|D]tfddDVq1dS)Nr[cr\rrr]r_rrr` raziterdicts..c3s|] }|VqdSr;)rIrJ)rrMrrrN%sziterdicts..)rRrbrrer[rW)rrrrrZrfr)rrrMrr,s  r,cOs,tt|}t||||g|Ri|dS)a Write a table in JSON format, with rows output as JSON objects. E.g.:: >>> import petl as etl >>> table1 = [['foo', 'bar'], ... ['a', 1], ... ['b', 2], ... ['c', 2]] >>> etl.tojson(table1, 'example.file3.json', sort_keys=True) >>> # check what it did ... print(open('example.file3.json').read()) [{"bar": 1, "foo": "a"}, {"bar": 2, "foo": "b"}, {"bar": 2, "foo": "c"}] Setting argument `lines` to `True` will enable to infer the writing format as a JSON lines . For more details about JSON lines please visit https://jsonlines.org/. >>> import petl as etl >>> table1 = [['name', 'wins'], ... ['Gilbert', [['straight', '7S'], ['one pair', '10H']]], ... ['Alexa', [['two pair', '4S'], ['two pair', '9S']]], ... ['May', []], ... ['Deloise',[['three of a kind', '5S']]]] >>> etl.tojson(table1, 'example.file3.jsonl', lines = True, sort_keys=True) >>> # check what it did ... print(open('example.file3.jsonl').read()) {"name": "Gilbert", "wins": [["straight", "7S"], ["one pair", "10H"]]} {"name": "Alexa", "wins": [["two pair", "4S"], ["two pair", "9S"]]} {"name": "May", "wins": []} {"name": "Deloise", "wins": [["three of a kind", "5S"]]} Note that this is currently not streaming, all data is loaded into memory before being written to the file. N)rb_dicts _writejson)tablerprefixsuffixrrobjrrrtojson(s ' rtFcOs:|rt|}ntt|}t||||g|Ri|dS)a" Write a table in JSON format, with rows output as JSON arrays. E.g.:: >>> import petl as etl >>> table1 = [['foo', 'bar'], ... ['a', 1], ... ['b', 2], ... ['c', 2]] >>> etl.tojsonarrays(table1, 'example.file4.json') >>> # check what it did ... print(open('example.file4.json').read()) [["a", 1], ["b", 2], ["c", 2]] Note that this is currently not streaming, all data is loaded into memory before being written to the file. N)rbr ro)rprrqrr output_headerrrrsrrr tojsonarraysVs   rvc Os|dd}t|i|}t|}|d=}tr$t||||||dntj|dddd}zt||||||dW|n |wWddSWddS1sVwYdS) NrFwb)rr r!Tr") rrr r&r _writeobjr'r(r-) rrsrqrrrrrencoderr.rrrrots    "rocCsz|dur |||r#|D]}||D]}||q|dq n ||D]}||q(|dur;||dSdS)N )write iterencode)ryrsr.rqrrrrecchunkrrrrxs    rxr?)NNN)NNNF)F)$ __future__rrrr'r*r5 json.encoderrosrtempfiler petl.compatrr petl.io.sourcesr r petl.util.baser r rrnrrrr:r8r7r)r,rtrvrorxrrrrs4     L E 8 +