o Uݢg&( @sdZddlmZmZmZddlZddlZddlZddlZddl Z ddl Z ddl m Z ddl mZmZmZmZdevr=daGdddeZGd d d Zd d Zd dZdFddZddZddZddZddZe jdrddl Z ddl!Z ddl"m#Z#e $e j%&dZ'e (dZ)ddZ*ndZ*ddddde*dddddddf d d!Z+dGd"d#Z,d$d%Z-d&d'Z.d(d)Z/d*d+Z0d,d-Z1d.d/Z2d0d1Z3d2d3Z4d4d5Z5d6d7Z6d8d9Z7d:d;Z8dd?Z:d@dAZ;dBdCZ5s z Record.items..)rrrrritems2s z Record.itemscCs t|S)zFormats the object as a string.)strrrrrr__str__9s zRecord.__str__ccs|jD]}t||VqdS)zwIterate through the values of the object corresponding to keys in the dictionary used to initialize the object.N)rr)rrrrr__iter__=s zRecord.__iter__cCst||j|S)zKGet the value associated with the Nth key in the source dictionary.)rr)rindexrrr __getitem__CszRecord.__getitem__cCsdS)z-This exists only for backwards compatibility.Nrrrrrcoerce_stringsIszRecord.coerce_stringsN) r r rrrrrr r"r#rrrrr(s rcCs|jD]}t||dqdS)zSet all of the outs to None.N)rr)outsrrrrclearMs r%cCs|}t|trt|s|tdks|tdkrd}|St|tr4i}|D] }t||||<q'|St|tr< |St|trJ|j ddd}|St|t rUt|}|St |dradd |D}|S) z;Converts NaN values into None values, and decode raw bytes.z+Infz-InfNutf-8ignore)errorsr cSsg|]}t|qSr) json_sanitize)rdrrr jsz!json_sanitize..) isinstancefloatmathisnandictrr)rbytesdecoderhasattr)dataZretvalkrrrr)Ts, "      r)cCstjt||dS)zSReturns a formatted json string of the data, with NaN values converted to None.)indent)jsondumpsr))r4r6rrrjson_dumps_safensr9cCstttjjttjjS)zRGet the current max rss memory for this process and completed child processes.)maxresourceZ getrusageZ RUSAGE_SELFZ ru_maxrssZRUSAGE_CHILDRENrrrr get_mem_kbts  r<cCs |ddS)zConvert from gb to kb.ir)mem_gbrrrconvert_gb_to_kb~s r>cCs4t|tt|}|dkrd|t|St|S)zEPad a string with leading spaces to be the same length as field_name.r )lenr)rvalueoffsetrrr padded_printsrCcCstdusJtj||S)zJAdd a function to the set of functions to be covered by the line profiler.N)r funcsappend)funcrrrprofiles  rGZlinux)SIGKILLccCs&td}tttt|||dS)zWhen used as the preexec_fn argument for subprocess.Popen etc, causes the subprocess to receive SIGKILL if the parent process terminates.rN)ctypesZc_ulong_LIBCZprctl_PR_SET_PDEATHSIGrH)Zzerorrrchild_preexec_set_pdeathsigs rNFcCsFtdusJtjdd|tj|||||||||| | | | | dS)zLog opening of a subprocess.Nexecr?) bufsize executablestdinstdoutstderr preexec_fn close_fdsshellcwdenvuniversal_newlines startupinfo creationflags)r metadatalogjoin subprocessPopen)argsrPrQrRrSrTrUrVrWrXrYrZr[r\rrrras$ racCs6tdusJtjdd|tj|||||tdS)zLog running a given subprocess.NrOr?)rWrRrSrTrU)r r]r^r_r` check_callrN)rbrRrSrTrWrrrrcs rccCs2t|tr |d}tdusJtjtjj|S)z#Get the file path for a named file.r&N) r,rencoder ospathr_r] files_path)filenamerrr make_paths   ricC&tdusJtjdus JtjjdS)z!Get the args from the invocation.Nrbr jobinfo invocationrrrrget_invocation_args  rncCrj)z-Get the call information from the invocation.Ncallrkrrrrget_invocation_callrorqcCrj)z)Get the martian version from the jobinfo.Nmartianr rlversionrrrrget_martian_versionrorucCrj)z+Get the pipelines version from the jobinfo.N pipelinesrsrrrrget_pipelines_versionrorwcC"tdusJtjdus JtjjS)z?Get the number of threads allocated to this job by the runtime.N)r rlthreadsrrrrget_threads_allocation  rzcCrx)zDGet the amount of memory in GB allocated to this job by the runtime.N)r rlr=rrrrget_memory_allocationr{r|cCstdp tdp dS)zpGet the UUID of the top-level pipeline instance. Returns an empty string if the UUID is not available. ZMRO_UUIDZMRO_FORCE_UUID)regetenvrrrrget_pipestance_uuidsrcCtdusJtj|dS)zUpdates the current progress of the stage, which will be displayed to the user (in the mrp log) next time mrp reads the file.N)r r]progressmessagerrrupdate_progress#s rcCtdusJtjd|dS)zLog a message.Ninfor r]r^rrrrlog_info+ rcCr)zLog a warning.Nwarnrrrrrlog_warn2rrcCr)zLog a timestamp for an action.Ntimerrrrrlog_time9rrcCs(tdusJtjdt||ddS)zLog an object in json format.Nr7)labelobject)r r]r^r9)robjrrrlog_json@s rcCst|)zRaise a stage exception.)r rrrrthrowIsrcCs$tdusJtj|tdS)z$Fail the pipeline with an assertion.N)r r] write_assertdonerrrrexitOs   rcCr)zAdd a message to the alarms.N)r r]alarmrrrrrWs rcCs$ddl}|ddd||dgdatS)z%Initialize with a fake test metadata.rNmainr}T)Z martian_shell StageWrapperr )rfZmr_shellrrrtest_initializecs r)N)NNNF)>r __future__rrrr7r.rer;r`sysZpathlibrtypingrrrr globalsr Exceptionr rr%r)r9r<r>rCrGplatform startswithrKZ ctypes.utilsignalrHZCDLLutilZ find_libraryrLZc_intrMrNrarcrirnrqrurwrzr|rrrrrrrrrrrrrrsz  %        '