B
    è?F[3#  ã               @   s°   d dl mZ G dd„ deƒZG dd„ deƒZG dd„ deƒZG dd	„ d	eƒZG d
d„ deƒZG dd„ deƒZG dd„ deƒZ	G dd„ deƒZ
G dd„ de
ƒZG dd„ de
ƒZdS )é    )Úsixc               @   s(   e Zd ZdZdd„ Zdd„ Zdd„ ZdS )	ÚStepz!
    Jobflow Step base class
    c             C   s
   t ƒ ‚dS )z=
        :rtype: str
        :return: URI to the jar
        N)ÚNotImplemented)Úself© r   ú,lib/python3.7/site-packages/boto/emr/step.pyÚjar   s    zStep.jarc             C   s
   t ƒ ‚dS )zS
        :rtype: list(str)
        :return: List of arguments for the step
        N)r   )r   r   r   r   Úargs%   s    z	Step.argsc             C   s
   t ƒ ‚dS )zB
        :rtype: str
        :return: The main class name
        N)r   )r   r   r   r   Ú
main_class,   s    zStep.main_classN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r	   r
   r   r   r   r   r      s   r   c               @   s2   e Zd ZdZddd„Zdd„ Zdd	„ Zd
d„ ZdS )ÚJarStepz
    Custom jar step
    NÚTERMINATE_JOB_FLOWc             C   s4   || _ || _|| _|| _t|tjƒr*|g}|| _dS )a  
        A elastic mapreduce step that executes a jar

        :type name: str
        :param name: The name of the step
        :type jar: str
        :param jar: S3 URI to the Jar file
        :type main_class: str
        :param main_class: The class to execute in the jar
        :type action_on_failure: str
        :param action_on_failure: An action, defined in the EMR docs to
            take on failure.
        :type step_args: list(str)
        :param step_args: A list of arguments to pass to the step
        N)ÚnameÚ_jarÚ_main_classÚaction_on_failureÚ
isinstancer   Ústring_typesÚ	step_args)r   r   r   r
   r   r   r   r   r   Ú__init__8   s    zJarStep.__init__c             C   s   | j S )N)r   )r   r   r   r   r   S   s    zJarStep.jarc             C   s   g }| j r| | j ¡ |S )N)r   Úextend)r   r	   r   r   r   r	   V   s    zJarStep.argsc             C   s   | j S )N)r   )r   r   r   r   r
   ^   s    zJarStep.main_class)Nr   N)r   r   r   r   r   r   r	   r
   r   r   r   r   r   4   s    
r   c            	   @   s:   e Zd ZdZddd„Zdd„ Zd	d
„ Zdd„ Zdd„ ZdS )ÚStreamingStepz
    Hadoop streaming step
    Nr   ú3/home/hadoop/contrib/streaming/hadoop-streaming.jarc             C   sX   || _ || _|| _|| _|| _|| _|| _|	| _|
| _|| _	t
|tjƒrN|g}|| _dS )a„  
        A hadoop streaming elastic mapreduce step

        :type name: str
        :param name: The name of the step
        :type mapper: str
        :param mapper: The mapper URI
        :type reducer: str
        :param reducer: The reducer URI
        :type combiner: str
        :param combiner: The combiner URI. Only works for Hadoop 0.20
            and later!
        :type action_on_failure: str
        :param action_on_failure: An action, defined in the EMR docs to
            take on failure.
        :type cache_files: list(str)
        :param cache_files: A list of cache files to be bundled with the job
        :type cache_archives: list(str)
        :param cache_archives: A list of jar archives to be bundled with
            the job
        :type step_args: list(str)
        :param step_args: A list of arguments to pass to the step
        :type input: str or a list of str
        :param input: The input uri
        :type output: str
        :param output: The output uri
        :type jar: str
        :param jar: The hadoop streaming jar. This can be either a local
            path on the master node, or an s3:// URI.
        N)r   ÚmapperÚreducerÚcombinerr   Úcache_filesÚcache_archivesÚinputÚoutputr   r   r   r   r   )r   r   r   r   r   r   r   r    r   r!   r"   r   r   r   r   r   f   s    #zStreamingStep.__init__c             C   s   | j S )N)r   )r   r   r   r   r   ™   s    zStreamingStep.jarc             C   s   d S )Nr   )r   r   r   r   r
   œ   s    zStreamingStep.main_classc             C   s  g }| j r| | j ¡ | d| jg¡ | jr<| d| jg¡ | jrT| d| jg¡ n| ddg¡ | jr¤t| jtƒr”x.| jD ]}| d|f¡ q|W n| d| jf¡ | jrº| d| jf¡ | j	rÞx| j	D ]}| d|f¡ qÈW | j
rx| j
D ]}| d	|f¡ qîW |S )
Nz-mapperz	-combinerz-reducerz-jobconfzmapred.reduce.tasks=0z-inputz-outputz
-cacheFilez-cacheArchive)r   r   r   r   r   r!   r   Úlistr"   r   r    )r   r	   r!   Z
cache_fileZcache_archiver   r   r   r	   Ÿ   s.    zStreamingStep.argsc             C   s<   d| j j| j j| j| j| j| j| j| j| j	| j
| j| jf S )Nz‰%s.%s(name=%r, mapper=%r, reducer=%r, action_on_failure=%r, cache_files=%r, cache_archives=%r, step_args=%r, input=%r, output=%r, jar=%r))Ú	__class__r   r   r   r   r   r   r   r    r   r!   r"   r   )r   r   r   r   Ú__repr__Ä   s
    zStreamingStep.__repr__)	NNr   NNNNNr   )	r   r   r   r   r   r   r
   r	   r%   r   r   r   r   r   b   s       
/%r   c                   s    e Zd ZdZ‡ fdd„Z‡  ZS )ÚScriptRunnerStepzEs3n://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jarc                s   t t| ƒj|| jf|Ž d S )N)Úsuperr&   r   ÚScriptRunnerJar)r   r   Úkw)r$   r   r   r   Ð   s    zScriptRunnerStep.__init__)r   r   r   r(   r   Ú__classcell__r   r   )r$   r   r&   Ì   s   r&   c               @   s   e Zd ZdddgZdS )ÚPigBasez4s3n://us-east-1.elasticmapreduce/libs/pig/pig-scriptz--base-pathz*s3n://us-east-1.elasticmapreduce/libs/pig/N)r   r   r   ÚBaseArgsr   r   r   r   r+   Ô   s   r+   c                   s&   e Zd ZdZdZd‡ fdd„	Z‡  ZS )ÚInstallPigStepz!
    Install pig on emr step
    zInstall PigÚlatestc                sD   g }|  | j¡ |  dg¡ |  d|g¡ tt| ƒj| j|d d S )Nz--install-pigz--pig-versions)r   )r   r,   r'   r-   r   ÚInstallPigName)r   Úpig_versionsr   )r$   r   r   r   á   s
    zInstallPigStep.__init__)r.   )r   r   r   r   r/   r   r*   r   r   )r$   r   r-   Ú   s   r-   c                   s&   e Zd ZdZdg f‡ fdd„	Z‡  ZS )ÚPigStepz
    Pig script step
    r.   c                sR   g }|  | j¡ |  d|g¡ |  ddd|g¡ |  |¡ tt| ƒj||d d S )Nz--pig-versionsz--run-pig-scriptz--argsz-f)r   )r   r,   r'   r1   r   )r   r   Zpig_filer0   Zpig_argsr   )r$   r   r   r   î   s    
zPigStep.__init__)r   r   r   r   r   r*   r   r   )r$   r   r1   é   s   r1   c               @   s   e Zd ZdddgZdS )ÚHiveBasez6s3n://us-east-1.elasticmapreduce/libs/hive/hive-scriptz--base-pathz+s3n://us-east-1.elasticmapreduce/libs/hive/N)r   r   r   r,   r   r   r   r   r2   ÷   s   r2   c                   s&   e Zd ZdZdZd‡ fdd„	Z‡  ZS )ÚInstallHiveStepz"
    Install Hive on EMR step
    zInstall Hiver.   Nc                s\   g }|  | j¡ |  dg¡ |  d|g¡ |d k	rB|  d| g¡ tt| ƒj| j|d d S )Nz--install-hivez--hive-versionsz--hive-site=%s)r   )r   r,   r'   r3   r   ÚInstallHiveName)r   Úhive_versionsZ	hive_siter   )r$   r   r   r     s    zInstallHiveStep.__init__)r.   N)r   r   r   r   r4   r   r*   r   r   )r$   r   r3   ý   s   r3   c                   s"   e Zd ZdZd‡ fdd„	Z‡  ZS )ÚHiveStepz
    Hive script step
    r.   Nc                sZ   g }|  | j¡ |  d|g¡ |  ddd|g¡ |d k	rB|  |¡ tt| ƒj||d d S )Nz--hive-versionsz--run-hive-scriptz--argsz-f)r   )r   r,   r'   r6   r   )r   r   Z	hive_filer5   Z	hive_argsr   )r$   r   r   r     s    
zHiveStep.__init__)r.   N)r   r   r   r   r   r*   r   r   )r$   r   r6     s    r6   N)Zboto.compatr   Úobjectr   r   r   r&   r+   r-   r1   r2   r3   r6   r   r   r   r   Ú<module>   s   .j