o
    _~#gQ                     @  s  d Z ddlmZ ddlmZ ddlZddlZddlZddlmZ ddlZddl	Z	ddl
Z
ddlmZ ddlZddlZddlZddlmZ ddlZddlZddlZddlZddlmZmZ dd	lmZ dd
lmZ ddlm  m Z! ddl"Z#ddl"m$Z$m%Z%m&Z&m'Z' ddl(m)Z* ddl+m,Z, ddl-m	  m.Z/ ddl0m1Z1m2Z2 dd Z3ej45ddde6de7de8dedg de7d9dde7d9ddddd e:dj;ddde:dj;ddde:dj;ddddddddf gdd Z<dd  Z=d!d" Z>d#d$ Z?dqd(d)Z@ej45d*ejAe>d+d,ejAe#jBd-d,ejAeje#jBejCd.d/d,ejAeje#jBd0d.d1d,ejAeje#jBd2d.d3d,gej45d4e#jBe>gej45d5e@e, d6d7 ZDd8d9 ZEd:d; ZFejGd<d= ZHG d>d? d?ZIG d@dA dAZJej45dBdCgdDe%g dEfdFe#jKLg dEg dGgfgdHdI ZMdJdK ZNej45dLg dMdNdO ZOdPdQ ZPG dRdS dSejQZRdTdU ZSdVdW ZTdXdY ZUdZd[ ZVej45d\d]d^d\ed_d`d`dagdbdc ZWddde ZXdfdg ZYe!jZdhdi Z[ej45djej\ejCgdkdl Z]dmdn Z^dodp Z_dS )ra<  
manage legacy pickle tests

How to add pickle tests:

1. Install pandas version intended to output the pickle.

2. Execute "generate_legacy_storage_files.py" to create the pickle.
$ python generate_legacy_storage_files.py <output_dir> pickle

3. Move the created pickle to "data/legacy_pickle/<version>" directory.
    )annotations)arrayN)partial)Path)Any)get_lzma_fileis_platform_little_endian)import_optional_dependency)flatten_buffer)	DataFrameIndexSeriesperiod_range)create_pickle_data)DayMonthEndc                 C  s   t |trt||  d S |drt| | d S |dkr4|tju r,| tju s*J d S | |ks2J d S ttd| dtj	}|| | d S )Nsp_	timestampassert__equal)

isinstancer   tmassert_index_equal
startswithassert_equalpdNaTgetattrassert_almost_equal)resultexpectedtyp
comparator r#   /var/www/static.ux5.de/https/Moving-Object-Detection-with-OpenCV/env/lib/python3.10/site-packages/pandas/tests/io/test_pickle.pycompare_element>   s   


r%   datas   123s   123456I         B)r+   r*   r*      )r+      C)orderFc                 C  s   t | }t| d}||ksJ t| ttfr || u sJ d S t|trA|jdks,J |jdks3J |js8J |j	|j
fksCJ d S d S )NAr)   r,   )r
   
memoryviewtobytesr   bytes	bytearrayndimformat
contiguousshapenbytes)r&   r   r    r#   r#   r$   test_flatten_bufferT   s   

r<   c           
   	   C  s  t  std ttjdD ]}| |}t|}|	 D ]\}}|	 D ]\}}|| | }|dkr|dkrt
|| |jj|jjksIJ |jjjrPJ t
|dk|dk |jj}|td tdksjJ |tjdd }	t|	tjszJ |	tjddd	ksJ |tjdd
 }	t|	tjsJ |	tjdddksJ q'|dkr|dkrt
|| t|jtsJ |jt ksJ |jdksJ t
|d|d q'|dkr|dv rt
|| q'|dkr|dv rt
|| q't||| q'qqd S )Nz"known failure on non-little endianzdata/legacy_pickle/*/*.p*kl*seriestsr   r)   r*   )hours)daysr?   )nanoseconds)r@   rA   indexperiodM)dt_tzcatframe)dt_mixed_tzs
cat_onecolcat_and_float)r   pytestskipr   __file__parentglobr   read_pickleitemsr   assert_series_equalrB   freq	normalizer   	Timedeltar   r   r   freqstrshiftassert_frame_equalr%   )
datapathlegacy_pickler&   r!   dvdtr   r    rS   resr#   r#   r$   test_picklesq   sF   

r^   c                 C  @   t |d}tj| |dd W d    d S 1 sw   Y  d S Nwbprotocolopenpickledumpobjpathfhr#   r#   r$   python_pickler      "rm   c                 C  sB   t | d}|d t|W  d    S 1 sw   Y  d S )Nrbr   )rf   seekrg   load)rk   rl   r#   r#   r$   python_unpickler   s   
$rr   dictreturnlist[tuple[str, Any]]c                 C  s   dd |   D S )zFlatten create_pickle_datac                 S  s&   g | ]\}}|  D ]}||fq
qS r#   )values).0r!   examplesexampler#   r#   r$   
<listcomp>   s    zflatten.<locals>.<listcomp>)rQ   )r&   r#   r#   r$   flatten   s   r{   pickle_writerpython)idpandas_proto_defaultrc   pandas_proto_highestr.   pandas_proto_4   pandas_proto_5writerztyp, expectedc              	   C  s   t  i}||| t|}t|||  t|}t|||  t|dd}||| |d W d    n1 s:w   Y  t|dd}t|}|d W d    n1 sZw   Y  t|||  W d    d S 1 spw   Y  d S )Nra   moder   ro   )r   ensure_cleanr   rP   r%   rr   rf   rp   )r!   r    r|   r   rk   r   handler#   r#   r$   test_round_trip_current   s    




"r   c                  C  `   t dtdd ttdtdtdd tdD tdd	} t	| j
tj}t| | d S )
N皙?x      r.   ABCDdtypec                 S     g | ]}d | qS zi-r#   rw   ir#   r#   r$   rz          z,test_pickle_path_pathlib.<locals>.<listcomp>r   columnsrB   )r   nparangereshaper   listobjectranger   round_trip_pathlib	to_pickler   rP   rX   dfr   r#   r#   r$   test_pickle_path_pathlib      r   c                  C  r   )
Nr   r   r   r   r   c                 S  r   r   r#   r   r#   r#   r$   rz      r   z.test_pickle_path_localpath.<locals>.<listcomp>r   r   )r   r   r   r   r   r   r   r   r   round_trip_localpathr   r   rP   rX   r   r#   r#   r$   test_pickle_path_localpath   r   r   c                   C  s   dt   dS )N__z	__.pickle)uuiduuid4r#   r#   r#   r$   get_random_path   s   r   c                   @  sT   e Zd ZejZdd Zdd Zej	
dg ddd Zd	d
 Zdd Zdd ZdS )TestCompressionc           	   	   C  s  |d u rt || d S |dkrt|d}n|dkr"t|d}n|dkrJtj|dtjd}|	|t
j| W d    n1 sDw   Y  nb|dkrt|d/}tj|dd}||t
j|}||| W d    n1 svw   Y  W d    n1 sw   Y  n!|d	krt |d}n|d
krtd|d}n	d| }t||dvrt|d*}| |	|  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S d S )Ngzipwbz2zipcompressiontarro   r   xzzstd	zstandardra   zUnrecognized compression type: )r   r   )shutilcopyfiler   rf   r   BZ2FilezipfileZipFileZIP_DEFLATEDwriteosrk   basenametarfile
gettarinfoaddfiler   r	   
ValueErrorread)	selfsrc_path	dest_pathr   frl   r   tarinfomsgr#   r#   r$   compress_file   sH   
"zTestCompression.compress_filec                 C  sL  |}|d }|d }t |}t |o}tdtdd ttdtdtdd	 t	d
D tdd}|j
||d t j||d$}	t|d}
|
|	  W d    n1 s\w   Y  W d    n1 skw   Y  tj|d d}t || W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )N.compressed.rawr   r   r   r   r   c                 S  r   r   r#   r   r#   r#   r$   rz   '  r   z7TestCompression.test_write_explicit.<locals>.<listcomp>r   r   r   ra   )r   r   r   r   r   r   r   r   r   r   r   decompress_filerf   r   r   r   rP   rX   )r   r   r   basepath1path2p1p2r   r   rl   df2r#   r#   r$   test_write_explicit  s&   Pz#TestCompression.test_write_explicitr   ) Nonebad7zc              	   C  s   t jtddM t|/}tdtdd t	t
dtdt	dd	 td
D tdd}|j||d W d    n1 s?w   Y  W d    d S W d    d S 1 sWw   Y  d S )NzUnrecognized compression typematchr   r   r   r   r   c                 S  r   r   r#   r   r#   r#   r$   rz   >  r   z;TestCompression.test_write_explicit_bad.<locals>.<listcomp>r   r   r   )rK   raisesr   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   rk   r   r#   r#   r$   test_write_explicit_bad7  s   "z'TestCompression.test_write_explicit_badc                 C  sX  |}|| }|d }| j | }t|}t|m}tdtdd t	t
dtdt	dd td	D tdd
}	|	| tj||d$}
t|d}||
  W d    n1 sbw   Y  W d    n1 sqw   Y  tj|d d}t|	| W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr   r   r   r   r   r   c                 S  r   r   r#   r   r#   r#   r$   rz   L  r   z4TestCompression.test_write_infer.<locals>.<listcomp>r   r   r   ra   )_extension_to_compressiongetlowerr   r   r   r   r   r   r   r   r   r   r   r   rf   r   r   r   rP   rX   )r   compression_extr   r   r   r   r   r   r   r   r   rl   r   r#   r#   r$   test_write_inferB  s(   
Pz TestCompression.test_write_inferc           
   	   C  s   |}|d }|d }t |b}t |D}tdtdd ttdtdtdd	 t	d
D tdd}|j
|d d | j|||d tj||d}	t ||	 W d    n1 s\w   Y  W d    d S W d    d S 1 stw   Y  d S )Nr   r   r   r   r   r   r   c                 S  r   r   r#   r   r#   r#   r$   rz   e  r   z6TestCompression.test_read_explicit.<locals>.<listcomp>r   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   rP   rX   )
r   r   r   r   r   r   r   r   r   r   r#   r#   r$   test_read_explicit\  s   Pz"TestCompression.test_read_explicitc              	   C  s  |}|d }|| }| j | }t|`}t|B}tdtdd t	t
dtdt	dd td	D tdd
}	|	j|d d | j|||d t|}
t|	|
 W d    n1 sbw   Y  W d    d S W d    d S 1 szw   Y  d S )Nr   r   r   r   r   r   c                 S  r   r   r#   r   r#   r#   r$   rz   |  r   z3TestCompression.test_read_infer.<locals>.<listcomp>r   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rP   rX   )r   r   r   r   r   r   r   r   r   r   r   r#   r#   r$   test_read_inferr  s   
PzTestCompression.test_read_inferN)__name__
__module____qualname__icomextension_to_compressionr   r   r   rK   markparametrizer   r   r   r   r#   r#   r#   r$   r      s    

r   c                   @  s&   e Zd Zejdg ddd ZdS )TestProtocolrd   )rb   r   r)   r*   c                 C  s   t |;}tdtdd ttdtdtdd t	dD tdd	}|j
||d
 t|}t || W d    d S 1 sCw   Y  d S )Nr   r   r   r   r   c                 S  r   r   r#   r   r#   r#   r$   rz     r   z*TestProtocol.test_read.<locals>.<listcomp>r   r   rc   )r   r   r   r   r   r   r   r   r   r   r   r   rP   rX   )r   rd   r   rk   r   r   r#   r#   r$   	test_read  s   
"zTestProtocol.test_readN)r   r   r   rK   r   r   r   r#   r#   r#   r$   r     s    r   pickle_fileexcolsztest_py27.pkl)abcztest_mi_py27.pkl)r2   r,   r/   c                 C  s*   | ddd|}t |}t|j| d S )Nior&   rg   )r   rP   r   r   r   )rY   r   r   rk   r   r#   r#   r$   test_unicode_decode_error  s   
r   c               	   C  s   t  c} tdtdd ttdtdtdd t	dD tdd	}t
| d
}|| W d    n1 s:w   Y  t
| d}t|}W d    n1 sTw   Y  t || W d    d S 1 sjw   Y  d S )Nr   r   r   r   r   c                 S  r   r   r#   r   r#   r#   r$   rz     r   z0test_pickle_buffer_roundtrip.<locals>.<listcomp>r   r   ra   ro   )r   r   r   r   r   r   r   r   r   r   rf   r   r   rP   rX   )rk   r   rl   r   r#   r#   r$   test_pickle_buffer_roundtrip  s   
"r   mockurl)zhttp://url.comzftp://test.comzhttp://gzip.comc                   s   dd }G dd d t  F fdd}tdtdd	 ttd
tdtdd t	dD tdd}|| | 
d| t|}t || W d    d S 1 sXw   Y  d S )Nc                 S  r_   r`   re   ri   r#   r#   r$   rm     rn   z3test_pickle_generalurl_read.<locals>.python_picklerc                   @  s6   e Zd ZdddZdd Zdd Zd	d
 Zdd ZdS )z5test_pickle_generalurl_read.<locals>.MockReadResponsert   r   c                 S  s0   t |d| _d|v rddi| _d S ddi| _d S )Nro   r   zContent-Encodingr   )rf   fileheaders)r   rk   r#   r#   r$   __init__  s   z>test_pickle_generalurl_read.<locals>.MockReadResponse.__init__c                 S  s   | S Nr#   r   r#   r#   r$   	__enter__     z?test_pickle_generalurl_read.<locals>.MockReadResponse.__enter__c                 W  s   |    d S r   )close)r   argsr#   r#   r$   __exit__  s   z>test_pickle_generalurl_read.<locals>.MockReadResponse.__exit__c                 S  
   | j  S r   )r   r   r   r#   r#   r$   r        
z:test_pickle_generalurl_read.<locals>.MockReadResponse.readc                 S  r   r   )r   r   r   r#   r#   r$   r     r  z;test_pickle_generalurl_read.<locals>.MockReadResponse.closeNrt   r   )r   r   r   r   r   r   r   r   r#   r#   r#   r$   MockReadResponse  s    
r  c                    s    S r   r#   )r   kwargsr  rk   r#   r$   mock_urlopen_read  s   z6test_pickle_generalurl_read.<locals>.mock_urlopen_readr   r   r   r   r   c                 S  r   r   r#   r   r#   r#   r$   rz     r   z/test_pickle_generalurl_read.<locals>.<listcomp>r   r   zurllib.request.urlopen)r   r   r   r   r   r   r   r   r   r   setattrr   rP   rX   )monkeypatchr   rm   r  r   r   r#   r  r$   test_pickle_generalurl_read  s   


"r	  c                  C  s   t d t ; d} tdtdd tt	dt
dtdd	 td
D t
dd}||  t| }t|| W d    d S 1 sGw   Y  d S )Nfsspeczmemory://mockfiler   r   r   r   r   c                 S  r   r   r#   r   r#   r#   r$   rz     r   z0test_pickle_fsspec_roundtrip.<locals>.<listcomp>r   r   )rK   importorskipr   r   r   r   r   r   r   r   r   r   r   r   rP   rX   )r   r   r   r#   r#   r$   test_pickle_fsspec_roundtrip  s   



"r  c                   @  s   e Zd ZdddZdS )MyTzrt   r   c                 C  s   d S r   r#   r   r#   r#   r$   r     r   zMyTz.__init__Nr  )r   r   r   r   r#   r#   r#   r$   r    s    r  c                  C  sD   t tdt f} t| }t|d | d  t|d ts J d S )Nr   r   r)   )r   r   r  r   round_trip_picklerR   r   )r    r   r#   r#   r$   test_read_pickle_with_subclass  s   
r  c                 C  s   t dtdd ttdtdtdd tdD tdd	}t	 }|j
|| d
 t| }W d   n1 s<w   Y  t }|j
|| d
 |d | |ks]| dv s]J tj|| d
}|d t|| dS )ze
    Read/write from binary file-objects w/wo compression.

    GH 26237, GH 29054, and GH 29570
    r   r   r   r   r   c                 S  r   r   r#   r   r#   r#   r$   rz     r   z9test_pickle_binary_object_compression.<locals>.<listcomp>r   r   r   Nr   )r   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   
read_bytesr   BytesIOrp   getvaluer   rP   rX   )r   r   rk   	referencebufferread_dfr#   r#   r$   %test_pickle_binary_object_compression  s    


r  c                 C  s8   | }|}dd }|| ||j  || ||j  d S )Nc                 S  s   t | }t | | d S r   )r   r  rX   )rG   	unpickledr#   r#   r$   _test_roundtrip7     
zDtest_pickle_dataframe_with_multilevel_index.<locals>._test_roundtrip)T)/multiindex_year_month_day_dataframe_random_data multiindex_dataframe_random_dataymdrG   r  r#   r#   r$   +test_pickle_dataframe_with_multilevel_index0  s   
r  c                  C  sH   t dddd} ttjdt| | }t|}|j	j
dks"J d S )Nz1/1/2011z1/1/2012rD   )rS   r*   )r   r   r   randomdefault_rngstandard_normallenr   r  rB   rV   )prngr>   new_tsr#   r#   r$   "test_pickle_timeseries_periodindexA  s   
r%  namei	  g     H@i     )r)   r*   c                 C  s0   t ttjdtjd| d}|j| ksJ d S )N
   r   )r&  )r   r  r   r   r   float64r&  )r&  r  r#   r#   r$   test_pickle_preserve_nameI  s   r*  c                 C     t | }t ||  d S r   r   r  rR   )datetime_seriesunp_tsr#   r#   r$   test_pickle_datetimesQ  r  r/  c                 C  r+  r   r,  )string_series
unp_seriesr#   r#   r$   test_pickle_stringsV  r  r2  c                  C  sf   t tddjdg } t| }|jjd jdksJ |jjd j	dks(J t
|dg |  d S )Nabccategoryr   r)   )r)   T)r   r   astypeilocr   r  _mgrblocksr7   r:   rR   )serr]   r#   r#   r$    test_pickle_preserves_block_ndim[  s
   
r:  rd   c                 C  s>   t td}tt|j| |dttj|d}t|| d S )Ni )rd   r   r   )	r   r   r   r   r   r   r   rP   rX   )rd   r   r   r   r#   r#   r$   %test_pickle_big_dataframe_compressionh  s   r;  c                 C  sf   | t tjdddd}t|d}t|}W d    n1 s w   Y  tg g d}t|| d S )Nr&   rZ   z1.2.4zempty_frame_v1_2_4-GH#42345.pklro   )rB   r   )	r   rM   rN   rf   rg   rq   r   r   rX   )rY   rk   fdr   r    r#   r#   r$   #test_pickle_frame_v124_unpickle_130s  s   r=  c                  C  s\   t dg di} d}tjt|d t }| |d W d    d S 1 s'w   Y  d S )Nr   r(   zpStarting with pandas version 3.0 all arguments of to_pickle except for the argument 'path' will be keyword-only.r   infer)r   r   assert_produces_warningFutureWarningr   r  r   )r   r   r  r#   r#   r$    test_pickle_pos_args_deprecation  s   "rA  )r&   rs   rt   ru   )`__doc__
__future__r   r   r   datetime	functoolsr   r   r   r   pathlibr   rg   r   r   typingr   r   r   numpyr   rK   pandas.compatr   r   pandas.compat._optionalr	   pandas.compat.compressorsr
   pandas.util._test_decoratorsutil_test_decoratorstdpandasr   r   r   r   r   pandas._testing_testingr   -pandas.tests.io.generate_legacy_storage_filesr   pandas.io.commoncommonr   pandas.tseries.offsetsr   r   r%   r   r   r6   r3   PickleBuffercastr   r   r<   r^   rm   rr   r{   paramr   HIGHEST_PROTOCOLr   r   r   fixturer   r   r   
MultiIndexfrom_arraysr   r   r	  r  tzinfor  r  r  r  r%  r*  r/  r2  skip_array_manager_invalid_testr:  DEFAULT_PROTOCOLr;  r=  rA  r#   r#   r#   r$   <module>   s    	$
2
	

 


)	



