o gz@s&dZddlmZddlZddlZddlmZddlZddlZddl m Z m Z m Z m Z mZddlZddlZddlZddlmZmZmZmZmZddlmZmZmZmZmZddlZddl Z ddl!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'dd l(m)Z)m*Z*dd l+m,Z,dd l-m.Z.e*Z/e0eeeZ1e12d ej3Gd ddZ4ej3GdddZ5dUddZ6dVddZ7dWddZ8 dXdYdd Z9d!d"Z:dZd%d&Z; '  ( d[d\d0d1Zd^d;d<Z?d_d>d?Z@    @  d`dadEdFZAGdGdHdHe jBe ZCGdIdJdJejDZEdbdOdPZFdcdQdRZGdddSdTZHdS)ezCommon IO api utilities) annotationsN)abc)BufferedIOBaseBytesIO RawIOBaseStringIO TextIOWrapper)IOAnyAnyStrMappingcast)urljoinurlparse uses_netloc uses_params uses_relative)BufferCompressionDictCompressionOptions FileOrBufferFilePathOrBufferStorageOptions) get_lzma_file import_lzma)import_optional_dependency) is_file_likec@s>eZdZUdZded<ded<ded<ded<d Zd ed <d S) IOArgsa) Return value of io/common.py:_get_filepath_or_buffer. Note (copy&past from io/parsers): filepath_or_buffer can be Union[FilePathOrBuffer, s3fs.S3File, gcsfs.GCSFile] though mypy handling of conditional imports is difficult. See https://github.com/python/mypy/issues/1297 rfilepath_or_bufferstrencodingmoder compressionFbool should_closeN)__name__ __module__ __qualname____doc____annotations__r%r+r+N/home/ubuntu/cloudmapper/venv/lib/python3.10/site-packages/pandas/io/common.pyr;s  rc@sleZdZUdZded<ded<ejedZded<d Z d ed <d Z d ed <dddZ dddZ dddZ dS) IOHandlesau Return value of io/common.py:get_handle Can be used as a context manager. This is used to easily close created buffers and to handle corner cases when TextIOWrapper is inserted. handle: The file handle to be used. created_handles: All file handles that are created by get_handle is_wrapped: Whether a TextIOWrapper needs to be detached. rhandlerr#)default_factoryz list[Buffer]created_handlesFr$ is_wrappedis_mmapreturnNonec Csx|jrt|jts J|j|j|j|jz |jD]}|q Wn t t fy3Ynwg|_d|_dS)z Close all created buffers. Note: If a TextIOWrapper was inserted, it is flushed and detached to avoid closing the potentially user-created buffer. FN) r1 isinstancer.rflushdetachr0removecloseOSError ValueError)selfr.r+r+r,r9bs     zIOHandles.closecC|SNr+r<r+r+r, __enter__vzIOHandles.__enter__argsr cGs |dSr>)r9)r<rBr+r+r,__exit__ys zIOHandles.__exit__Nr3r4)r3r-)rBr r3r4)r&r'r(r)r* dataclassesfieldlistr0r1r2r9r@rCr+r+r+r,r-Ms      r-r3r$cCst|tsdSt|jtvS)z Check to see if a URL has a valid protocol. Parameters ---------- url : str or unicode Returns ------- isurl : bool If `url` has a valid protocol return True otherwise False. F)r5r parse_urlscheme _VALID_URLSurlr+r+r,is_url}s rMrFileOrBuffer[AnyStr]cCst|tr tj|S|S)a] Return the argument with an initial component of ~ or ~user replaced by that user's home directory. Parameters ---------- filepath_or_buffer : object to be converted if possible Returns ------- expanded_filepath_or_buffer : an expanded filepath or the input if not expandable )r5r ospath expanduser)rr+r+r, _expand_users  rRr4cCst|tr tddS)NzPassing a bool to header is invalid. Use header=None for no header or header=int or list-like of ints to specify the row(s) making up the column names)r5r$ TypeError)headerr+r+r,validate_header_args rUFFilePathOrBuffer[AnyStr]convert_file_likecCs6|s t|r ttt|St|tjr|}t|S)a Attempt to convert a path-like object to a string. Parameters ---------- filepath_or_buffer : object to be converted Returns ------- str_filepath_or_buffer : maybe a string version of the object Notes ----- Objects supporting the fspath protocol (python 3.6+) are coerced according to its __fspath__ method. Any other object is passed through unchanged, which includes bytes, strings, buffers, or anything else that's not even path-like. ) rr rr r5rOPathLike __fspath__rR)rrWr+r+r,stringify_paths  rZcOsddl}|jj|i|S)z` Lazy-import wrapper for stdlib urlopen, as that imports a big chunk of the stdlib. rN)urllib.requestrequesturlopen)rBkwargsurllibr+r+r,r]sr]rLrcCst|tod|vo|d S)zR Returns true if the given URL looks like something fsspec can handle z://)zhttp://zhttps://)r5r startswithrKr+r+r, is_fsspec_urls  rautf-8rr!r r#rr"storage_optionsrc Cst|}t|\}}t||}|r$t|dr$d|vr$tjdtddd}t||d}|dur6|dd  }d |vrM|d vrM|d vrMt|d |t |}d|vr[d|vr[|d7}t |t rt |r|pgi}ddl}|jj||d}t|} | jdd} | dkrddi}t| } Wdn1swYt| ||d|dSt|r4t |t sJ|dr|dd}|dr|dd}td} g} ztdddlm}m}||tg} Wn tyYnwz| j|fd|i|pi}Wn.t | y*|durddi}nt|}d|d<| j|fd|i|p#i}Ynwt|||d|dS|r;t!dt |t t"t#j#frPtt$|||d|dSt%|s`d t&|}t!|t|||d|dS)!a If the filepath_or_buffer is a url, translate and return the buffer. Otherwise passthrough. Parameters ---------- filepath_or_buffer : a url, filepath (str, py.path.local or pathlib.Path), or buffer compression : {{'gzip', 'bz2', 'zip', 'xz', None}}, optional encoding : the encoding to use to decode bytes, default is 'utf-8' mode : str, optional storage_options : dict, optional Extra options that make sense for a particular storage connection, e.g. host, port, username, password, etc., if using a URL that will be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". An error will be raised if providing this argument with a local path or a file-like buffer. See the fsspec and backend storage implementation docs for the set of allowed keys and values .. versionadded:: 1.2.0 ..versionchange:: 1.2.0 Returns the dataclass IOArgs. writebzDcompression has no effect when passing a non-binary object as input.) stacklevelN)method_-w)bz2xz)zutf-16zutf-32z( will not write the byte order mark for tr)headerszContent-EncodinggzipriT)rr!r#r%r"zs3a://zs3://zs3n://fsspecbotocore) ClientErrorNoCredentialsErrorr"anonz?storage_options passed with file object or non-fsspec file pathFz)Invalid file path or buffer object type: )'rZget_compression_methodinfer_compressionhasattrwarningswarnRuntimeWarningdictreplacelowerUnicodeWarningr5r rMr[r\Requestr]rpgetrreadrrar`rbotocore.exceptionsrtruPermissionError ImportErroropentupler;bytesmmaprRrtype)rr!r#r"rdcompression_method fsspec_moder_req_inforeqcontent_encodingreaderrrerr_types_to_retry_with_anonrtrufile_objmsgr+r+r,_get_filepath_or_buffers!               rrPcCsddlm}td||S)z converts an absolute native path to a FILE URL. Parameters ---------- path : a path in native format Returns ------- a valid FILE URL r) pathname2urlzfile:)r[rr)rPrr+r+r,file_path_to_urls rz.gzz.bz2z.zipz.xz)rqrmziprn"tuple[str | None, CompressionDict]c CsZt|tr%t|}z |d}W||fSty$}ztd|d}~wwi}|}||fS)a Simplifies a compression argument to a compression method string and a mapping containing additional arguments. Parameters ---------- compression : str or mapping If string, specifies the compression method. If mapping, value at key 'method' specifies compression method. Returns ------- tuple of ({compression method}, Optional[str] {compression arguments}, Dict[str, Any]) Raises ------ ValueError on mapping missing 'method' key riz.If mapping, compression must have key 'method'N)r5r r}popKeyErrorr;)r#compression_argsrerrr+r+r,rws   rw str | NonecCs|durdS|dkr-t|dd}t|tsdStD]\}}||r*|SqdS|tvr3|Sddgtt}d|d|}t|)a Get the compression method for filepath_or_buffer. If compression='infer', the inferred compression method is returned. Otherwise, the input compression method is returned unchanged, unless it's invalid, in which case an error is raised. Parameters ---------- filepath_or_buffer : str or file handle File path or object. compression : {'infer', 'gzip', 'bz2', 'zip', 'xz', None} If 'infer' and `filepath_or_buffer` is path-like, then detect compression from the following extensions: '.gz', '.bz2', '.zip', or '.xz' (otherwise no compression). Returns ------- string or None Raises ------ ValueError on invalid compression specified. NinferT)rWUnrecognized compression type: z Valid compression types are ) rZr5r _compression_to_extensionitemsrendswithsortedr;)rr# extensionvalidrr+r+r,rxs*  rxT path_or_buf memory_mapis_texterrorscCs|pd}t||rd|vr|d7}t|tr|}|dvr*td|dddt|||||d}|j} t| ||j|j ||j d t v\} }} t| t} t |j } | d }|r|j d d |_ |d kr| r|t| tsoJtjd| |j d | } nstjd| |j d| } nf|dkrtj| fd|j i| } nU|dkrt| |j fi| } | j dkr| | | } t| dkr| | } n+t| dkrtd|td| n|dkrtt| |j } n d|}t|t| trJ| | n't| tr#|jrd|j vrt| |j |j|d d} nt| |j } | | d}|rK|s2t| |j rKt| |j|d d} | | t|jtpI|j }| |jrbt|jtr\J| |jt| trjJt| | |||j dS)a Get file handle for given path/buffer and mode. Parameters ---------- path_or_buf : str or file handle File path or object. mode : str Mode to open path_or_buf with. encoding : str or None Encoding to use. compression : str or dict, default None If string, specifies compression mode. If dict, value at key 'method' specifies compression mode. Compression mode must be one of {'infer', 'gzip', 'bz2', 'zip', 'xz', None}. If compression mode is 'infer' and `filepath_or_buffer` is path-like, then detect compression from the following extensions: '.gz', '.bz2', '.zip', or '.xz' (otherwise no compression). If dict and compression mode is one of {'zip', 'gzip', 'bz2'}, or inferred as one of the above, other entries passed as additional compression options. .. versionchanged:: 1.0.0 May now be a dict with key 'method' as compression mode and other keys as compression options if compression mode is 'zip'. .. versionchanged:: 1.1.0 Passing compression options as keys in dict is now supported for compression modes 'gzip' and 'bz2' as well as 'zip'. memory_map : bool, default False See parsers._parser_params for more information. is_text : bool, default True Whether the type of the content passed to the file/buffer is string or bytes. This is not the same as `"b" not in mode`. If a string content is passed to a binary file/buffer, a wrapper is inserted. errors : str, default 'strict' Specifies how encoding and decoding errors are to be handled. See the errors argument for :func:`open` for a full list of options. storage_options: StorageOptions = None Passed to _get_filepath_or_buffer .. versionchanged:: 1.2.0 Returns the dataclass IOHandles rbrf) Nstrictignorer~xmlcharrefreplacebackslashreplace namereplacesurrogateescape surrogatepassz%Invalid value for `encoding_errors` (z). Please see z=https://docs.python.org/3/library/codecs.html#error-handlers zfor valid values.)r!r#r"rdrirorrq)filenamer")fileobjr"rmr"rrcrzZero files found in ZIP file z9Multiple files found in ZIP file. Only one file per ZIP: rnrr!rnewlineF)r.r0r1r2r#Nr+)_is_binary_moder5r rr;rr_maybe_memory_mapr!r"r#rr}rr~rqGzipFilermBZ2File _BytesZipFileappendnamelistlenrrlzmarr%reverser-)rr"r!r#rrrrdioargsr.handlesis_pathr zip_namesrr1r+r+r, get_handle s<                 rcsTeZdZdZ ddfd d Zd d Zdfdd ZfddZeddZ Z S)ra  Wrapper for standard library class ZipFile and allow the returned file-like handle to accept byte strings via `write` method. BytesIO provides attributes of file-like object and ZipFile.writestr writes bytes strings into a member of the archive. Nfilerr"r archive_namerc sF|dd}||_d|_dtji}||tj||fi|dS)Nrfrr#)r~rmultiple_write_bufferzipfile ZIP_DEFLATEDupdatesuper__init__)r<rr"rr^ kwargs_zip __class__r+r,rs   z_BytesZipFile.__init__cCs2|jdurt|tr tnt|_|j|dSr>)rr5rrrre)r<datar+r+r,res z_BytesZipFile.writer3r4csf|jdus |jjr dS|jp|jpd}|jt||jWddS1s,wYdS)Nr)rclosedrrrwritestrgetvalue)r<rrr+r,r6s "z_BytesZipFile.flushcs|tdSr>)r6rr9r?rr+r,r9'sz_BytesZipFile.closecCs |jduSr>)fpr?r+r+r,r+s z_BytesZipFile.closedr>)rrr"r rrrD) r&r'r(r)rrer6r9propertyr __classcell__r+r+rr,rs  rc@sLeZdZdZ   ddd d ZdddZd ddZd!d"ddZd#ddZdS)$ _MMapWrappera Wrapper for the Python's mmap class so that it can be properly read in by Python's csv.reader class. Parameters ---------- f : file object File object to be mapped onto memory. Must support the 'fileno' method or have an equivalent attribute rbrTfr r!r rdecoder$cCsn||_||_t||d|_||_i|_dD]}t||sqt|||j|<qt j | dt j d|_ dS)N)r)seekablereadable writeabler)access) r!rcodecsgetincrementaldecoderdecoderr attributesrygetattrrfileno ACCESS_READ)r<rr!rr attributer+r+r,r=s z_MMapWrapper.__init__namecs$jvr fddStjS)Ncs jSr>)rr+rr<r+r,Rs z*_MMapWrapper.__getattr__..)rrr)r<rr+rr, __getattr__Ps  z_MMapWrapper.__getattr__r3cCr=r>r+r?r+r+r,__iter__UrAz_MMapWrapper.__iter__sizeint str | bytescCs:|j|}|jr|dkpt||k}|jj||dS|S)Nr)final)rrrrr)r<rcontentrr+r+r,rXs z_MMapWrapper.readcCs,|j}|j|}|dkrt|dS)Nr )rreadlinerr StopIterationlstrip)r<newbytesrr+r+r,__next__bs   z_MMapWrapper.__next__N)rbrT)rr r!r rr rr$)rr )r3r)r)rrr3r)r3r ) r&r'r(r)rrrrrr+r+r+r,r0s    rr.rr'tuple[FileOrBuffer, bool, list[Buffer]]cCsg}|t|dp t|tM}|s|||fSt|tr4|r*d|vr*t||||dd}nt||}||zttjt||||}|| ||||}Wn t y\d}Ynw|||fS)zTry to memory map file/buffer.rrfrrF) ryr5r rrr rrr9r8 Exception)r.rr!r"rrrwrappedr+r+r,rss,           rc CsFd}t|}t|ts |Sz tj|}W|Sttfy"Y|Sw)zTest whether file exists.F)rZr5r rOrPexistsrSr;)rrr+r+r, file_existss rcCs`d|vsd|vr d|vStjtjtjtjf}tt||rdStt f}t ||p/dt |d|vS)z+Whether the handle is opened in binary moderorfFr") r StreamWriter StreamReaderStreamReaderWritertempfileSpooledTemporaryFile issubclassrrrr5r)r.r" text_classesbinary_classesr+r+r,rsr)r3r$)rrNr3rNrD)F)rrVrWr$r3rN)rLrr3r$)rbNrcN) rrr!r r#rr"r rdrr3r)rPr r3r )r#rr3r)rrr#rr3r)NNFTNN)rrr"r r!rr#rrr$rr$rrrdrr3r-)r.rrr$r!r r"r rrrr$r3r)rrr3r$)r.rr"r r3r$)Ir) __future__rrmr collectionsrrErqiorrrrrrrOrtypingr r r r r urllib.parserrrHrrrrzrpandas._typingrrrrrr pandas.compatrrpandas.compat._optionalrpandas.core.dtypes.commonrrsetrJdiscard dataclassrr-rMrRrUrZr]rarrrrwrxrZipFilerIteratorrrrrr+r+r+r,sp       /   "   5  #> n9 C -