diff --git a/examples/__pycache__/task_gromacs.cpython-38.pyc b/examples/__pycache__/task_gromacs.cpython-38.pyc index 064c0e2..8429bd3 100644 Binary files a/examples/__pycache__/task_gromacs.cpython-38.pyc and b/examples/__pycache__/task_gromacs.cpython-38.pyc differ diff --git a/examples/know_my_cli_adv.py b/examples/know_my_cli_adv.py index 8134a5d..00b22bf 100644 --- a/examples/know_my_cli_adv.py +++ b/examples/know_my_cli_adv.py @@ -10,14 +10,14 @@ def know_my_cli(ctl, host, user, key, password_file): https://superuser.com/questions/1524610/detect-if-apt-get-update-is-necessary ''' - ctl.runtime_initer('host',host,str) - ctl.runtime_initer('user',user,str) - ctl.runtime_initer('key',key,str) + ctl.rit('host',host,str) + ctl.rit('user',user,str) + ctl.rit('key',key,str) ctl.init_cd('./cli/') def load_pass(x): with open(password_file,'r') as f: v = yaml.safe_load(f.read()) - ctl.runtime_setter("passwds",v,object) + ctl.rset("passwds",v,object) ctl.RWC(run=load_pass) @@ -29,7 +29,7 @@ def load_pass(x): ctl.lazy_git_url_commit('https://github.com/shouldsee/pype', '598b7a2b1201d138260c22119afd7b4d5449fe97', 'temp_pype') - SH_CONN = RO(ctl).rundir +'/connect-' + ctl.runtime['key'] +'.sh' + SH_CONN = ctl.ro.rundir +'/connect-' + ctl.R['key'] +'.sh' ctl.RWC(run = lambda rt: open( SH_CONN(),'w').write( f''' diff --git a/examples/task_gromacs.py b/examples/task_gromacs.py index 3f9dfce..0158a5f 100644 --- a/examples/task_gromacs.py +++ b/examples/task_gromacs.py @@ -310,7 +310,7 @@ def write_dep_file(ctx): ctl.RWC(run='echo "19\n0\n" | {GMX} energy -f npt.edr -o pressure.xvg') ctl.RWC(run='echo "25\n0\n" | {GMX} energy -f npt.edr -o density.xvg') - ctl.lazy_grace_png( 'density.xvg.png') + ctl.lazy_grace_png( 'density.xvg.png' ) ctl.RWC(run='''{GMX} grompp -f md.mdp -c npt.gro -t npt.cpt -p topol.top -o md_0_1.tpr''') ctl.RWC(CWST,'md_0_1.gro',run='{GMX} mdrun -deffnm md_0_1 -nb gpu') diff --git a/pype/__pycache__/controller.cpython-38.pyc b/pype/__pycache__/controller.cpython-38.pyc index 009a6b8..9f995e1 100644 Binary files a/pype/__pycache__/controller.cpython-38.pyc and b/pype/__pycache__/controller.cpython-38.pyc differ diff --git a/pype/controller.py b/pype/controller.py index fed6f29..eb43b5e 100644 --- a/pype/controller.py +++ b/pype/controller.py @@ -22,27 +22,43 @@ def namedlist(*args,**kw): # from collections import OrderedDict,namedtuple from ._internals import make_SafeOrderedDict,SafeOrderedDict from filelock import FileLock -def s(cmd,shell=True,raw=False): + +class PypeBase(object): + @property + def ro(self):return RO(self) + +def EPRINT(i=0): + v = i //100 + if v==0: + # if i==0: + # return eprint + + return sys.stderr.write + elif v==1: + return sys.stdout.write + +def s(cmd:str,shell=True,raw=False): if 'set -e' not in cmd: cmd = 'set -e; ' + cmd ''' Might stuck ''' + EPRINT()(f'[cmd]{cmd}\n') proc = subprocess.Popen(cmd, shell=shell, # stderr=subprocess.PIPE, ### problematic stderr=subprocess.STDOUT, stdout=subprocess.PIPE, executable='/bin/bash', # encoding='utf-8' - ) + ) ret = io.StringIO() while proc.poll() is None: text = proc.stdout.readline(1024) if not raw: text = text.decode() - sys.stdout.write(str(text)) + EPRINT(110)(str(text)) ret.write(str(text)) # text = proc.stderr.readline(1024) @@ -357,6 +373,7 @@ def __getitem__(self,key): caller = (lambda x,key=key: x.__getitem__(RO(key,None, frame)() ) ) return RuntimeObject(self, caller, frame) + def __lt__(self,b): frame = FRAME(1) caller = lambda x: x.__lt__(RO(b,None,frame)()) @@ -872,7 +889,7 @@ class ValueNotReadyError(ValueError): # x = AppendTypeChecker(x,ControllerNode) # return x.chain_with(ParentBuiltChecker) -class PlaceHolder(object): +class PlaceHolder(PypeBase): # @classmethod def __init__(self,name,value=NotInitObject): self.name = name @@ -905,14 +922,7 @@ def set_pdb(self): self.use_pdb = 1 return self - - # def __new__(cls,name): - # self = super().__new__() - # self.name = name - # self.value = NotInitObject - # return self - # return import functools @@ -924,7 +934,8 @@ def _getattr(obj, attr): return getattr(obj, attr, *args) return functools.reduce(_getattr, [obj] + attr.split('.')) -class Controller(object): + +class Controller(PypeBase): def __init__(self): @@ -942,6 +953,7 @@ def __init__(self): self.init_cd(None) self.is_built= False self._is_compiled = False + @classmethod def from_func(cls, f, *args,**kwargs): ctl = cls() @@ -951,8 +963,13 @@ def from_func(cls, f, *args,**kwargs): def use_pdb(self): self._use_pdb = 1 - def __getitem__(self,k): - return self._state.__getitem__(k) + # def __getitem__(self,k): + # return self._state.__getitem__(k) + # class _RuntimeView(object) + @property + def R(self): + return self.runtime + def export(self, k, v, t=object): self._state[k] = ControllerNode( @@ -1043,16 +1060,19 @@ def run_node_with_control(self, control, check_ctx, run, import pdb;pdb.set_trace() - def msg(head): + def msg(ep,head): f, filename, lineno = stack_ele co = f.f_code - eprint('') - eprint(f'{head}(name={name!r}, code {co.co_name!r}, file={co.co_filename!r}, line {lineno!r})') - eprint(f' File "{filename!r}", line {lineno}, in {co.co_name})') + # eprint = EPRINT(1) + ep('\n') + ep(f'{head}(name={name!r}, code {co.co_name!r},' + f' file={co.co_filename!r}, line {lineno!r})\n') + ep(f' File "{filename!r}", line {lineno}, in {co.co_name})\n') print_tb_stacks_2([stack_ele]) - msg ('[BULD]') - eprint('[CHCK]',end='') + #### important running information + msg (EPRINT(10),'[BULD]') + EPRINT(10)('[CHCK]') check = RuntimeObject(check).call() ''' Dangerous and needs a way to chain to the left? @@ -1068,9 +1088,9 @@ def msg(head): # print(f'[SKIP]({name},{co.co_name},)') # print(f'[SKIP]({name},{stack_ele})') # msg('[SKIP]') - eprint('[SKIP]') + EPRINT(10)('[SKIP]\n') else: - eprint('[RUNN]') + EPRINT(10)('[RUNN]\n') # msg('[RUNN]') # print(f'[RUNN]({name},{stack_ele})') # run(runtime) @@ -1101,7 +1121,7 @@ def pprint_stats(self): # x.add_row(xx) x.add_row([rgetattr(v,xn) for xn in x.field_names]) x.align='l' - eprint(x.get_string()) + EPRINT(15)(x.get_string()+'\n') # pprint(self.meta) @property @@ -1155,6 +1175,7 @@ def runtime_initer(self, k, v, t=object): self._runtime.__setitem__(k,AppendTypeChecker(v,t,FRAME(1))) return (self,k,v,t) # return self._runtime + rit = runtime_initer # @property def runtime_setter(self,k,v,t=object): @@ -1167,6 +1188,7 @@ def runtime_setter(self,k,v,t=object): self._runtime_copy.__setitem__(k,AppendTypeChecker(v,t,FRAME(1))) return (self,k,v,t) # return self._runtime_copy + rset = runtime_setter def register_node(self, @@ -1240,6 +1262,7 @@ def build(self, rundir=None, metabase=None, runtime= None, os.makedirs(rundir) if not os.path.exists(rundir) else None self.rundir = rundir + self._runtime_copy['RUNDIR'] = rundir # rundir = RuntimeObject( (rundir, self._runtime), self.target_dir).call() @@ -1254,7 +1277,7 @@ def build(self, rundir=None, metabase=None, runtime= None, metabase = 'PYPE.json' meta_file = rundir +'/' +metabase - eprint(f'[AcquirngLock]{meta_file}') + EPRINT(5)(f'[AcquirngLock]{meta_file}\n') with FileLock(meta_file+'.lock'): self.meta = meta = PypeExecResultList(data=[]) diskmeta = PypeExecResultList(data=[]) @@ -1267,7 +1290,7 @@ def build(self, rundir=None, metabase=None, runtime= None, if not len(diskmeta.data): raise ValueError('Empty Json') except Exception as e: - eprint(f'[Controller.run] Unable to load metafile:{meta_file} {e}') + EPRINT(5)(f'[Controller.run] Unable to load metafile:{meta_file} {e}\n') # raise e with open(meta_file,'w',1) as f: @@ -1375,6 +1398,7 @@ def _lazy_wget(ctx,url=url,target=target): return self.register_node( check_write_2, check_ctx=target, run=_lazy_wget, name=name, + frame=FRAME(1), built=target,) def lazy_apt_install(self, PACK,name=None): @@ -1407,8 +1431,9 @@ def lazy_pip_install(self, TARGET_LIST,pre_cmd='', # frame=FRAME(1)) - def lazy_git_url_commit(self, url, commit, target_dir=None, - name=lambda x:f'lazy_git/{x}', + def lazy_git_url_commit( + self, url, commit, target_dir=None, + name=lambda x:f'lazy_git/{x}', ): ''' fix cwd at runtime diff --git a/setup.py b/setup.py index c549a8d..cef392f 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ import os,glob,sys config = dict( name='pype', - version = '0.0.4', + version = '0.0.5', packages=['pype'], include_package_data=True, license='MIT',