
    sKg+A                     <   d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlmZmZ ddlmZ ddlmZmZmZ ddlmZ dd	lmZ  ej6                  d
d      Z G d de      Zd Z G d de      Z  G d de!      Z" G d de e"      Z#y)z.Storage providers backends for Memory caching.    )PicklingErrorN)ABCMetaabstractmethod   )concurrency_safe_rename)mkdirpmemstr_to_bytes
rm_subdirs)format_time)numpy_pickleCacheItemInfozpath size last_accessc                       e Zd ZdZy)CacheWarningz:Warning to capture dump failures except for PicklingError.N)__name__
__module____qualname____doc__     Z/home/alanp/www/video.onchill/myenv/lib/python3.12/site-packages/joblib/_store_backends.pyr   r      s    Dr   r   c                     t        t        j                               }dj                  ||t	        j
                               } || |       |S )z>Writes an object into a unique file in a concurrency-safe way.z{}.thread-{}-pid-{})id	threadingcurrent_threadformatosgetpid)object_to_writefilename
write_func	thread_idtemporary_filenames        r   concurrency_safe_writer#      sD    9++-.I.55)RYY[* 23r   c                       e Zd ZdZdZed        Zed        Zed        Zed        Z	ed        Z
ed        Zed	 e       fd
       Zy)StoreBackendBasezaHelper Abstract Base Class which defines all methods that
       a StorageBackend must implement.Nc                      y)a  Opens an item on the store and return a file-like object.

        This method is private and only used by the StoreBackendMixin object.

        Parameters
        ----------
        f: a file-like object
            The file-like object where an item is stored and retrieved
        mode: string, optional
            the mode in which the file-like object is opened allowed valued are
            'rb', 'wb'

        Returns
        -------
        a file-like object
        Nr   )selffmodes      r   
_open_itemzStoreBackendBase._open_item/       r   c                      y)a  Checks if an item location exists in the store.

        This method is private and only used by the StoreBackendMixin object.

        Parameters
        ----------
        location: string
            The location of an item. On a filesystem, this corresponds to the
            absolute path, including the filename, of a file.

        Returns
        -------
        True if the item exists, False otherwise
        Nr   r'   locations     r   _item_existszStoreBackendBase._item_existsB   r+   r   c                      y)a-  Moves an item from src to dst in the store.

        This method is private and only used by the StoreBackendMixin object.

        Parameters
        ----------
        src: string
            The source location of an item
        dst: string
            The destination location of an item
        Nr   )r'   srcdsts      r   
_move_itemzStoreBackendBase._move_itemS   r+   r   c                      y)zCreates a location on the store.

        Parameters
        ----------
        location: string
            The location in the store. On a filesystem, this corresponds to a
            directory.
        Nr   r-   s     r   create_locationz StoreBackendBase.create_locationa   r+   r   c                      y)zClears a location on the store.

        Parameters
        ----------
        location: string
            The location in the store. On a filesystem, this corresponds to a
            directory or a filename absolute path
        Nr   r-   s     r   clear_locationzStoreBackendBase.clear_locationl   r+   r   c                      y)zReturns the whole list of items available in the store.

        Returns
        -------
        The list of items identified by their ids (e.g filename in a
        filesystem).
        Nr   r'   s    r   	get_itemszStoreBackendBase.get_itemsw   r+   r   r   c                      y)a  Configures the store.

        Parameters
        ----------
        location: string
            The base location used by the store. On a filesystem, this
            corresponds to a directory.
        verbose: int
            The level of verbosity of the store
        backend_options: dict
            Contains a dictionary of named parameters used to configure the
            store backend.
        Nr   )r'   r.   verbosebackend_optionss       r   	configurezStoreBackendBase.configure   r+   r   )r   r   r   r   r.   r   r*   r/   r3   r5   r7   r:   dictr>   r   r   r   r%   r%   )   s    + H $            *+TV  r   r%   )	metaclassc                       e Zd ZdZddZddZd Zd Zd Zd Z	d	 Z
d
 Zd ZddZd Zd Zd Z	 ddZ	 ddZd Zd Zy)StoreBackendMixina  Class providing all logic for managing the store in a generic way.

    The StoreBackend subclass has to implement 3 methods: create_location,
    clear_location and configure. The StoreBackend also has to provide
    a private _open_item, _item_exists and _move_item methods. The _open_item
    method has to have the same signature as the builtin open and return a
    file-like object.
    Nc                    t        j                  j                  | j                  g| }|dkD  r|/dj	                  t        t        j                         |z
              nd}t         j                  j                  |d         }|Ad|v r=dj                  d |d   j                         D              }|d	j	                  |      z  }d
j	                  ||      }	|dk  rt        dj	                  |	             nt        dj	                  |	|             t        | d      sdn| j                  }
t         j                  j                  |d      }| j                  |      st        d|z        |
2| j                  |d      5 }t        j                   |      }ddd       |S t        j                   ||
      }|S # 1 sw Y   S xY w)z:Load an item from the store given its id as a list of str.r   Nz{: <16} r   
input_argsz, c              3   :   K   | ]  } d j                   |   yw)z{}={}N)r   .0items     r   	<genexpr>z.StoreBackendMixin.load_item.<locals>.<genexpr>   s$      #O/Mt $27>>4#8/Ms   z({})z[Memory]{}: Loading {}
   z{0}...z{0} from {1}	mmap_mode
output.pklzANon-existing item (may have been cleared).
File %s does not existrb)rL   )r   pathjoinr.   r   r   timebasenameitemsprinthasattrrL   r/   KeyErrorr*   r   load)r'   call_idr<   	timestampmetadata	full_path	ts_string	signaturekwargsmsgrL   r   r(   rI   s                 r   	load_itemzStoreBackendMixin.load_item   s   GGLL99	Q;%1 #))+diikI6M*NO79 ((4I#(@ #O/7/E/K/K/M#O OV]]622	*11)YGC|hooc*+n++C;<!({!;T.. 	 77<<	<8  * ?AIJ K K 40A#((+ 1   $$XCD	 1 s   F;;Gc                     	 t        j                  j                   j                  g| } j	                  |      s j                  |       t         j                  j                  |d      }|dkD  rt        d|z          fd} j                  |||       y# t        $ r(}t        j                  d| dt               Y d}~yd}~ww xY w)z;Dump an item in the store at the id given as a list of str.rM   rK   zPersisting in %sc                    j                  |d      5 }	 t        j                  | |j                         d d d        y # t        $ r(}t        j                  d| dt               Y d }~5d }~ww xY w# 1 sw Y   y xY w)Nwb)compresszjUnable to cache to disk: failed to pickle output. In version 1.5 this will raise an exception. Exception: .)r*   r   dumprd   r   warningswarnFutureWarning)to_writedest_filenamer(   er'   s       r   r    z/StoreBackendMixin.dump_item.<locals>.write_func   sx    __]D9Q	$))(AN :9 )  556Cq: *	  :9s.   A5"A	A2
A-(A5-A22A55A>z`Unable to cache to disk. Possibly a race condition in the creation of the directory. Exception: re   N)r   rO   rP   r.   r/   r5   rT   _concurrency_safe_write	Exceptionrg   rh   r   )r'   rX   rI   r<   	item_pathr   r    rl   s   `       r   	dump_itemzStoreBackendMixin.dump_item   s    	T]]=W=I$$Y/$$Y/ww||I|<H|(945 ((xD 	MM99:1> 	s   BB 	C$CCc                     t        j                  j                  | j                  g| }| j	                  |      r| j                  |       yy)z1Clear the item at the id, given as a list of str.Nr   rO   rP   r.   r/   r7   )r'   rX   ro   s      r   
clear_itemzStoreBackendMixin.clear_item   >    GGLL99	Y'	* (r   c                     t        j                  j                  | j                  g| }t         j                  j                  |d      }| j	                  |      S )z<Check if there is an item at the id, given as a list of str.rM   )r   rO   rP   r.   r/   )r'   rX   ro   r   s       r   contains_itemzStoreBackendMixin.contains_item   sB    GGLL99	77<<	<8  **r   c                 Z    dt        j                  j                  | j                  g| iS )zReturn information about item.r.   r   rO   rP   r.   r'   rX   s     r   get_item_infozStoreBackendMixin.get_item_info   #    BGGLLAABBr   c                 ^   	 t        j                  j                  | j                  g| }t         j                  j                  |d      }| j	                  |d      5 }t        j                  |j                         j                  d            cddd       S # 1 sw Y   yxY w#  i cY S xY w)z"Return actual metadata of an item.metadata.jsonrN   utf-8N)	r   rO   rP   r.   r*   jsonloadsreaddecode)r'   rX   ro   r   r(   s        r   get_metadatazStoreBackendMixin.get_metadata   sx    	T]]=W=Iww||I?H40Azz!&&(//'":; 100	Is*   AB& 2B	B& B#B& #B& &B,c                      	 t        j                  j                   j                  g| } j	                  |       t         j                  j                  |d      } fd} j                  |||       y#  Y yxY w)z Store metadata of a computation.r}   c                     j                  |d      5 }|j                  t        j                  |       j	                  d             d d d        y # 1 sw Y   y xY w)Nrc   r~   )r*   writer   dumpsencode)rj   rk   r(   r'   s      r   r    z4StoreBackendMixin.store_metadata.<locals>.write_func   s?    __]D9QGGDJJx077@A :99s   4AAN)r   rO   rP   r.   r5   rm   )r'   rX   rZ   ro   r   r    s   `     r   store_metadataz StoreBackendMixin.store_metadata   si    	T]]=W=I  +ww||I?HB ((8ZH	s   A3A7 7A;c                 x    t        j                  j                  | j                  g| }| j	                  |      S )z,Check cached function is available in store.)r   rO   rP   r.   object_existsr'   rX   	func_paths      r   contains_pathzStoreBackendMixin.contains_path  s.    GGLL99	!!),,r   c                     t        j                  j                  | j                  g| }| j	                  |      r| j                  |       yy)z0Clear all items with a common path in the store.Nrr   r   s      r   
clear_pathzStoreBackendMixin.clear_path  rt   r   c                 p   t        j                  j                  | j                  g| }| j	                  |      s| j                  |       |\t         j                  j                  |d      }| j                  |d      5 }|j                  |j                  d             ddd       yy# 1 sw Y   yxY w)&Store the code of the cached function.Nfunc_code.pyrc   r~   )	r   rO   rP   r.   r/   r5   r*   r   r   )r'   rX   	func_coder   r   r(   s         r   store_cached_func_codez(StoreBackendMixin.store_cached_func_code  s    GGLL99	  +  + ww||I~>H40A	((12 10 !00s   !B,,B5c                     t        j                  j                  | j                  g|d }	 | j	                  |d      5 }|j                         j                  d      cddd       S # 1 sw Y   yxY w#   xY w)r   r   rN   r~   N)r   rO   rP   r.   r*   r   r   )r'   rX   r   r(   s       r   get_cached_func_codez&StoreBackendMixin.get_cached_func_code  s^    77<<HHH	40Avvxw/ 100	s)   A5  A)	A5 )A2.A5 2A5 5A8c                 Z    dt        j                  j                  | j                  g| iS )z?Return information related to the cached function if it exists.r.   rx   ry   s     r   get_cached_func_infoz&StoreBackendMixin.get_cached_func_info&  r{   r   c                 :    | j                  | j                         y)zClear the whole store content.N)r7   r.   r9   s    r   clearzStoreBackendMixin.clear*  s    DMM*r   c                     | j                  |||      }|D ]G  }| j                  dkD  rt        dj                  |             	 | j	                  |j
                         I y# t        $ r Y Vw xY w)zX
        Remove the store's oldest files to enforce item, byte, and age limits.
        rK   zDeleting item {0}N)_get_items_to_deleter<   rT   r   r7   rO   OSError)r'   bytes_limititems_limit	age_limititems_to_deleterI   s         r   enforce_store_limitsz&StoreBackendMixin.enforce_store_limits.  st     33i
 $D||b )0067##DII.	 $
  
 s   A!!	A-,A-c                 B   t        |t              rt        |      }| j                         }|sg S t	        d |D              }|||z
  }nd}|t        |      |z
  }nd}|4t        d |D              }t        j                  j                         |z
  }	nd}	|dk  r|dk  r	|	|	kD  rg S |j                  t        j                  d             g }
d}d}|D ]E  }||k\  r||k\  r|	|	|j                  k  r |
S |
j                  |       ||j                  z  }|dz  }G |
S )zW
        Get items to delete to keep the store under size, file, & age limits.
        c              3   4   K   | ]  }|j                     y wN)sizerG   s     r   rJ   z9StoreBackendMixin._get_items_to_delete.<locals>.<genexpr>Q  s     /499   Nr   c              3   4   K   | ]  }|j                     y wr   )last_accessrG   s     r   rJ   z9StoreBackendMixin._get_items_to_delete.<locals>.<genexpr>^  s     @%$T--%r   r   )keyr   )
isinstancestrr	   r:   sumlenmindatetimenowsortoperator
attrgetterr   appendr   )r'   r   r   r   rS   r   to_delete_sizeto_delete_items
older_itemdeadliner   size_so_faritems_so_farrI   s                 r   r   z&StoreBackendMixin._get_items_to_deleteD  sS    k3')+6K I///"!K/NN"!%j;6OO @%@@J((,,.:HH aOq$8!Z(%:I 	

x**=9
:D. O3%D4D4D)D 	 ""4(499$KAL  r   c                 B    t        |||      }| j                  ||       y)z7Writes an object into a file in a concurrency-safe way.N)r#   r3   )r'   rj   r   r    r"   s        r   rm   z)StoreBackendMixin._concurrency_safe_write  s$    3H4<jJ*H5r   c                 d    dj                  | j                  j                  | j                        S )z/Printable representation of the store location.z#{class_name}(location="{location}"))
class_namer.   )r   	__class__r   r.   r9   s    r   __repr__zStoreBackendMixin.__repr__  s/    4;;~~.. < H 	Hr   )r   NN)r   r   )NN)r   r   r   r   r`   rp   rs   rv   rz   r   r   r   r   r   r   r   r   r   r   rm   r   r   r   r   rB   rB      sm     D>++C-
+	3C+
 <@. <@9v6Hr   rB   c                       e Zd ZdZ ee      Z eej                  j                        Z
 ee      Zd Zd Zd ZddZy)FileSystemStoreBackendz7A StoreBackend used with local or network file systems.c                 h    || j                   k(  rt        |       yt        j                  |d       y)zDelete location on store.T)ignore_errorsN)r.   r
   shutilrmtreer-   s     r   r7   z%FileSystemStoreBackend.clear_location  s$    %x MM($7r   c                     t        |       y)zCreate object location on storeN)r   r-   s     r   r5   z&FileSystemStoreBackend.create_location  s
    xr   c                    g }t        j                  | j                        D ]  \  }}}t        j                  dt         j
                  j                  |            }|s=t         j
                  j                  |d      }	 t         j
                  j                  |      }t        j                  j                  |      }	 |D cg c]"  }t         j
                  j                  ||      $ }	}t        d |	D              }
|j                  t        ||
|              |S # t        $ r5 	 t         j
                  j                  |      }n# t        $ r Y Y 3w xY wY w xY wc c}w # t        $ r Y Nw xY w)z7Returns the whole list of items available in the store.z[a-f0-9]{32}rM   c              3   Z   K   | ]#  }t         j                  j                  |       % y wr   )r   rO   getsize)rH   fns     r   rJ   z3FileSystemStoreBackend.get_items.<locals>.<genexpr>  s%      "<,:b #%''//""5,:s   )+)r   walkr.   rematchrO   rR   rP   getatimer   r   fromtimestampr   r   r   )r'   rS   dirpath_	filenamesis_cache_hash_diroutput_filenamer   r   full_filenamesdirsizes              r   r:   z FileSystemStoreBackend.get_items  s\   %'WWT]]%;!GQ	 ")+)9)9')B!D !"$'',,w"E!"$''"2"2?"CK '//==kJ	09&;09" ')ggll7B&?09 # &;! "<,:"< <G ]7G+68 97 &<< -  !!&(gg&6&6w&?" ! ! $!&;   	sZ    D?E$'E*E$	E(EE	EEEEEE$$	E10E1Nc                 L   |i }|| _         t        j                  j                  | j                         st	        | j                          |j                  dd      | _        |j                  d      }| j                  r|t        j                  dd       || _	        || _
        y)zsConfigure the store backend.

        For this backend, valid store options are 'compress' and 'mmap_mode'
        Nrd   FrL   zSCompressed items cannot be memmapped in a filesystem store. Option will be ignored.   )
stacklevel)r.   r   rO   existsr   getrd   rg   rh   rL   r<   )r'   r.   r<   r=   rL   s        r   r>   z FileSystemStoreBackend.configure  s    
 " O !ww~~dmm,4==! (++J> $''4	==Y2MM F%&( #r   )r   N)r   r   r   r   staticmethodopenr*   r   rO   r   r/   r   r3   r7   r5   r:   r>   r   r   r   r   r     sA    Ad#J/L56J8"Hr   r   )$r   pickler   r   r   os.pathr   r   r   rQ   rg   collectionsr   r   abcr   r   	backportsr   diskr   r	   r
   loggerr   rD   r   
namedtupler   Warningr   r#   r%   objectrB   r   r   r   r   <module>r      s    4   	 	          ' . 5 5  &&&'>@	7 	
f fRvH vHrO-/@ Or   