o ;DiC@sddlmZmZmZddlZddlmZmZddlm Z ddl m Z m Z dddZ Gdd d e Zd d Z  dd dZ  dddZ   dddZ   dddZGddde ZddZdS))absolute_importprint_functiondivisionN) string_typesizip) ArgumentError)TabledictscCst|||dS)a' Extract all documents from a Whoosh index. E.g.:: >>> import petl as etl >>> import os >>> # set up an index and load some documents via the Whoosh API ... from whoosh.index import create_in >>> from whoosh.fields import * >>> schema = Schema(title=TEXT(stored=True), path=ID(stored=True), ... content=TEXT) >>> dirname = 'example.whoosh' >>> if not os.path.exists(dirname): ... os.mkdir(dirname) ... >>> index = create_in(dirname, schema) >>> writer = index.writer() >>> writer.add_document(title=u"First document", path=u"/a", ... content=u"This is the first document we've added!") >>> writer.add_document(title=u"Second document", path=u"/b", ... content=u"The second one is even more interesting!") >>> writer.commit() >>> # extract documents as a table ... table = etl.fromtextindex(dirname) >>> table +------+-------------------+ | path | title | +======+===================+ | '/a' | 'First document' | +------+-------------------+ | '/b' | 'Second document' | +------+-------------------+ Keyword arguments: index_or_dirname Either an instance of `whoosh.index.Index` or a string containing the directory path where the index is stored. indexname String containing the name of the index, if multiple indexes are stored in the same directory. docnum_field If not None, an extra field will be added to the output table containing the internal document number stored in the index. The name of the field will be the value of this argument. ) indexname docnum_field) TextIndexViewindex_or_dirnamer r rG/var/www/Datamplify/venv/lib/python3.10/site-packages/petl/io/whoosh.py fromtextindex s0rc@seZdZdddZddZdS)r NcCs||_||_||_dSNr )selfrr r rrr__init__Cs zTextIndexView.__init__cCst|j|j|jSr) itertextindexrr r rrrr__iter__Hs zTextIndexView.__iter__NN__name__ __module__ __qualname__rrrrrrr As  r c cs(ddl}t|tr|}|jj||dd}d}nt||jjr$|}d}ntd|zazQ|durSt|j }|Vt j |j }| D] \} } || VqHn(|ft|j }|Vt j |j }| D] \} } | f|| VqnWnW|r|dSdS|r|ww)NrTr readonlyF"expected string or index, found %r) whoosh.index isinstancerindexopen_dirIndexrtupleschema stored_namesoperator itemgetterreader iter_docsclose) rr r whooshdirnamer" needs_closinghdrastuple_stored_fields_dictdocnumrrrrMsH     rFc Csddl}ddl}t|tr|}|jj|||d}d} nt||jjr'|}d} ntd||} z/zt |D] } | j di| q7| j |||j j dWn| W| r_|dSdS| rh|ww) a Load all rows from `table` into a Whoosh index. N.B., this will clear any existing data in the index before loading. E.g.:: >>> import petl as etl >>> import datetime >>> import os >>> # here is the table we want to load into an index ... table = (('f0', 'f1', 'f2', 'f3', 'f4'), ... ('AAA', 12, 4.3, True, datetime.datetime.now()), ... ('BBB', 6, 3.4, False, datetime.datetime(1900, 1, 31)), ... ('CCC', 42, 7.8, True, datetime.datetime(2100, 12, 25))) >>> # define a schema for the index ... from whoosh.fields import * >>> schema = Schema(f0=TEXT(stored=True), ... f1=NUMERIC(int, stored=True), ... f2=NUMERIC(float, stored=True), ... f3=BOOLEAN(stored=True), ... f4=DATETIME(stored=True)) >>> # load index ... dirname = 'example.whoosh' >>> if not os.path.exists(dirname): ... os.mkdir(dirname) ... >>> etl.totextindex(table, dirname, schema=schema) Keyword arguments: table A table container with the data to be loaded. index_or_dirname Either an instance of `whoosh.index.Index` or a string containing the directory path where the index is to be stored. schema Index schema to use if creating the index. indexname String containing the name of the index, if multiple indexes are stored in the same directory. merge Merge small segments during commit? optimize Merge all segments together? rN)r TFr)mergeoptimize mergetyper)r whoosh.writingr!rr" create_inr$rwriterr add_documentcommitwritingCLEARcancelr,) tablerr&r r5r6r-r.r"r/r:drrr totextindex}s<.      rBTc Csddl}t|tr|}|jj||dd}d}nt||jjr#|}d}ntd||} z0zt|D] } | j di| q3| j ||dWn t yR| wW|r\| dSdS|re| ww) aF Load all rows from `table` into a Whoosh index, adding them to any existing data in the index. Keyword arguments: table A table container with the data to be loaded. index_or_dirname Either an instance of `whoosh.index.Index` or a string containing the directory path where the index is to be stored. indexname String containing the name of the index, if multiple indexes are stored in the same directory. merge Merge small segments during commit? optimize Merge all segments together? rNFrTr)r5r6r)r r!rr"r#r$rr:r r;r< Exceptionr?r,) r@rr r5r6r-r.r"r/r:rArrrappendtextindexs8      rD c Cst||||||||dS)a Search a Whoosh index using a query. E.g.:: >>> import petl as etl >>> import os >>> # set up an index and load some documents via the Whoosh API ... from whoosh.index import create_in >>> from whoosh.fields import * >>> schema = Schema(title=TEXT(stored=True), path=ID(stored=True), ... content=TEXT) >>> dirname = 'example.whoosh' >>> if not os.path.exists(dirname): ... os.mkdir(dirname) ... >>> index = create_in('example.whoosh', schema) >>> writer = index.writer() >>> writer.add_document(title=u"Oranges", path=u"/a", ... content=u"This is the first document we've added!") >>> writer.add_document(title=u"Apples", path=u"/b", ... content=u"The second document is even more " ... u"interesting!") >>> writer.commit() >>> # demonstrate the use of searchtextindex() ... table1 = etl.searchtextindex('example.whoosh', 'oranges') >>> table1 +------+-----------+ | path | title | +======+===========+ | '/a' | 'Oranges' | +------+-----------+ >>> table2 = etl.searchtextindex('example.whoosh', 'doc*') >>> table2 +------+-----------+ | path | title | +======+===========+ | '/a' | 'Oranges' | +------+-----------+ | '/b' | 'Apples' | +------+-----------+ Keyword arguments: index_or_dirname Either an instance of `whoosh.index.Index` or a string containing the directory path where the index is to be stored. query Either a string or an instance of `whoosh.query.Query`. If a string, it will be parsed as a multi-field query, i.e., any terms not bound to a specific field will match **any** field. limit Return at most `limit` results. indexname String containing the name of the index, if multiple indexes are stored in the same directory. docnum_field If not None, an extra field will be added to the output table containing the internal document number stored in the index. The name of the field will be the value of this argument. score_field If not None, an extra field will be added to the output table containing the score of the result. The name of the field will be the value of this argument. fieldboosts An optional dictionary mapping field names to boosts. search_kwargs Any extra keyword arguments to be passed through to the Whoosh `search()` method. )limitr r score_field fieldboosts search_kwargsSearchTextIndexView)rqueryrFr r rGrHrIrrrsearchtextindexs JrMc Cst|||||||||d S)a) Search an index using a query, returning a result page. Keyword arguments: index_or_dirname Either an instance of `whoosh.index.Index` or a string containing the directory path where the index is to be stored. query Either a string or an instance of `whoosh.query.Query`. If a string, it will be parsed as a multi-field query, i.e., any terms not bound to a specific field will match **any** field. pagenum Number of the page to return (e.g., 1 = first page). pagelen Number of results per page. indexname String containing the name of the index, if multiple indexes are stored in the same directory. docnum_field If not None, an extra field will be added to the output table containing the internal document number stored in the index. The name of the field will be the value of this argument. score_field If not None, an extra field will be added to the output table containing the score of the result. The name of the field will be the value of this argument. fieldboosts An optional dictionary mapping field names to boosts. search_kwargs Any extra keyword arguments to be passed through to the Whoosh `search()` method. )pagenumpagelenr r rGrHrIrJ) rrLrNrOr r rGrHrIrrrsearchtextindexpageQs &rPc@s$eZdZ   dddZddZdS)rKNc Cs@||_||_||_||_||_||_||_||_| |_| |_ dSr) _index_or_dirname_query_limit_pagenum_pagelen _indexname _docnum_field _score_field _fieldboosts_search_kwargs) rrrLrFrNrOr r rGrHrIrrrrs zSearchTextIndexView.__init__c Cs.t|j|j|j|j|j|j|j|j|j |j Sr) itersearchindexrQrRrSrTrUrVrWrXrYrZrrrrrs  zSearchTextIndexView.__iter__)NNNNNNNNrrrrrrK~s  rKc cs(ddl} ddl} ddl} | st} t|tr%|} | jj| |dd} d} nt|| jjr1|} d} nt d|zzt }|durE||f7}|durN||f7}t | j }||7}|Vt|trt| j j| j | j |d}||}nt|| jjr|nt d|tj| j }| c}|dur|j|fd|i| }n |j||fd |i| }|dur|dur|D]}||Vqn,t||D]$\\}}}t }|dur||f7}|dur||f7}|||7}|VqWdn1swYWnW| r | dSdS| r| ww) NrTrFr)rHz/expected string or whoosh.query.Query, found %rrFrO)r whoosh.querywhoosh.qparserdictr!rr"r#r$rr%r&r'qparserMultifieldParsernamesparserLQueryr(r)searchersearch search_pageritemsr,)rrLrFrNrOr r rGrHrIr-r.r"r/r0r'parserr1rdresultsdocr4scorerowrrrr[s             r[r)NNFF)NTF)rENNNNN) __future__rrrr( petl.compatrr petl.errorsrpetl.util.baserr rr rrBrDrMrPrKr[rrrrs0  4 0 O 5 P -