Skip to content

Commit

Permalink
[Automated Commit] Format Codebase
Browse files Browse the repository at this point in the history
  • Loading branch information
@mlcommons-bot
mlcommons-bot committed Feb 2, 2025
1 parent 3bb9f63 commit 3d2520a
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 45 deletions.
23 changes: 12 additions & 11 deletions automation/cache/module_misc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,12 @@ def copy_to_remote(i):
remote_host = i.get('remote_host')
if not remote_host:
return {'return': 1,
'error': 'Please input remote host_name/IP via --remote_host'}
'error': 'Please input remote host_name/IP via --remote_host'}
remote_cm_repos_location = i.get(
'remote_cm_repos_location', os.path.join(
"/home", os.getlogin(), "CM", "repos"))
'remote_cm_repos_location', os.path.join(
"/home", os.getlogin(), "CM", "repos"))
remote_cm_cache_location = os.path.join(
remote_cm_repos_location, "local", "cache")
remote_cm_repos_location, "local", "cache")

remote_port = i.get('remote_port', '22')
remote_user = i.get('remote_user', os.getlogin())
Expand Down Expand Up @@ -91,28 +91,29 @@ def copy_to_remote(i):
cm_cached_state_json_file = os.path.join(path, "cm-cached-state.json")
if not os.path.exists(cm_cached_state_json_file):
return {'return': 1,
'error': f'cm-cached-state.json file missing in {path}'}
'error': f'cm-cached-state.json file missing in {path}'}

with open(cm_cached_state_json_file, "r") as f:
cm_cached_state = json.load(f)

new_env = cm_cached_state['new_env']
new_state = cm_cached_state['new_state'] # Todo fix new state
cm_repos_path = os.environ.get(
'CM_REPOS', os.path.join(
os.path.expanduser("~"), "CM", "repos"))
'CM_REPOS', os.path.join(
os.path.expanduser("~"), "CM", "repos"))
cm_cache_path = os.path.realpath(
os.path.join(cm_repos_path, "local", "cache"))

for key,val in new_env.items():
if type(val) == str and cm_cache_path in val:
new_env[key] = val.replace(cm_cache_path, remote_cm_cache_location)
for key, val in new_env.items():
if isinstance(val, str) and cm_cache_path in val:
new_env[key] = val.replace(
cm_cache_path, remote_cm_cache_location)

with open("tmp_remote_cached_state.json", "w") as f:
json.dump(cm_cached_state, f, indent=2)

remote_cached_state_file_location = os.path.join(
remote_cm_cache_location, cacheid, "cm-cached-state.json")
remote_cm_cache_location, cacheid, "cm-cached-state.json")
copy_cmd = f"rsync -avz -e 'ssh -p {remote_port}' tmp_remote_cached_state.json {remote_user}@{remote_host}:{remote_cached_state_file_location}"
print(copy_cmd)
os.system(copy_cmd)
Expand Down
27 changes: 14 additions & 13 deletions automation/experiment/module.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,8 +280,8 @@ def run(self, i):

explore_keys.append(k)

if type(v)!=list:
v=eval(v)
if not isinstance(v, list):
v = eval(v)

explore_dimensions.append(v)

Expand Down Expand Up @@ -443,8 +443,8 @@ def run(self, i):
all_results.append(result)

r = utils.save_json(
file_name=experiment_result_file,
meta=all_results)
file_name=experiment_result_file,
meta=all_results)
if r['return'] > 0:
return r

Expand Down Expand Up @@ -534,7 +534,8 @@ def replay(self, i):

if uid != '':
for d in datetimes:
r = self._find_uid({'path': experiment_path, 'datetime': d, 'uid': uid})
r = self._find_uid(
{'path': experiment_path, 'datetime': d, 'uid': uid})
if r['return'] > 0:
return r

Expand Down Expand Up @@ -731,11 +732,11 @@ def _find_or_add_artifact(self, i):
for e in lst:
print('{}) {}'.format(num, e.path))
print(
' Tags: {}'.format(
','.join(
e.meta.get(
'tags',
[]))))
' Tags: {}'.format(
','.join(
e.meta.get(
'tags',
[]))))
num += 1

if not console:
Expand Down Expand Up @@ -777,7 +778,7 @@ def _find_or_add_artifact(self, i):
return r

lst = r['list']
if len(lst) == 0 or len(lst) >1:
if len(lst) == 0 or len(lst) > 1:
return {
'return': 1, 'error': 'created experiment artifact with UID {} but can\'t find it - weird'.format(experiment_uid)}

Expand Down Expand Up @@ -833,7 +834,7 @@ def _find_uid(self, i):
if uid != '':
for result in meta:
ruid = result.get('uid', '').strip()
if ruid != '' and ruid ==uid:
if ruid != '' and ruid == uid:
rr['result'] = result
break

Expand All @@ -842,7 +843,7 @@ def _find_uid(self, i):
############################################################################


def flatten_dict(d, flat_dict= {}, prefix = ''):
def flatten_dict(d, flat_dict={}, prefix=''):

for k in d:
v = d[k]
Expand Down
49 changes: 28 additions & 21 deletions automation/utils/module_cfg.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ def load_cfg(i):
key_end = i.get('key_end', [])

ii = {'action': 'find',
'automation': 'cfg'}
'automation': 'cfg'}
if artifact != '':
ii['artifact'] = artifact
elif tags != '':
Expand Down Expand Up @@ -92,7 +92,7 @@ def load_cfg(i):
continue

if f.startswith('_') or (not f.endswith(
'.json') and not f.endswith('.yaml')):
'.json') and not f.endswith('.yaml')):
continue

if len(key_end) > 0:
Expand Down Expand Up @@ -130,11 +130,11 @@ def load_cfg(i):
add = False

if add and len(
prune_list) > 0 and uid not in prune_list:
prune_list) > 0 and uid not in prune_list:
add = False

if add and prune_key != '' and prune_key_uid != '' and prune_key_uid != meta.get(
prune_key, None):
prune_key, None):
add = False

if add:
Expand All @@ -152,7 +152,7 @@ def load_cfg(i):
if file_tags == '':
if name != '':
add_all_tags += [v.lower()
for v in name.split(' ')]
for v in name.split(' ')]
else:
add_all_tags += file_tags.split(',')

Expand Down Expand Up @@ -242,18 +242,19 @@ def process_base(meta, full_path):
else:
base[k] = meta[k]

elif type(v) == list:
elif isinstance(v, list):
for vv in v:
base[k].append(vv)
elif type(v) == dict:
elif isinstance(v, dict):
base[k].merge(v)

meta = base

return {'return': 0, 'meta':meta}
return {'return': 0, 'meta': meta}

##########################################################################


def select_cfg(i):

self_module = i['self_module']
Expand All @@ -263,7 +264,8 @@ def select_cfg(i):
title = i.get('title', '')

# Check if alias is not provided
r = self_module.cmind.access({'action': 'find', 'automation':'cfg', 'tags':'basic,docker,configurations'})
r = self_module.cmind.access(
{'action': 'find', 'automation': 'cfg', 'tags': 'basic,docker,configurations'})
if r['return'] > 0:
return r

Expand All @@ -277,18 +279,19 @@ def select_cfg(i):

if alias != '':
for ext in ['.json', '.yaml']:
p1 = os.path.join(p, alias +ext)
p1 = os.path.join(p, alias + ext)
if os.path.isfile(p1):
selector.append({'path': p1, 'alias':alias})
selector.append({'path': p1, 'alias': alias})
break

else:
files = os.listdir(p)

for f in files:
if not f.startswith('_cm') and (
f.endswith('.json') or f.endswith('.yaml')):
selector.append({'path': os.path.join(p, f), 'alias':f[:-5]})
f.endswith('.json') or f.endswith('.yaml')):
selector.append(
{'path': os.path.join(p, f), 'alias': f[:-5]})

# Load meta for name and UID
selector_with_meta = []
Expand All @@ -300,7 +303,7 @@ def select_cfg(i):
full_path_without_ext = path[:-5]

r = cmind.utils.load_yaml_and_json(full_path_without_ext)
if r['return'] >0:
if r['return'] > 0:
print('Warning: problem loading configuration file {}'.format(path))

meta = r['meta']
Expand All @@ -311,7 +314,7 @@ def select_cfg(i):

# Quit if no configurations found
if len(selector_with_meta) == 0:
return {'return': 16, 'error':'configuration was not found'}
return {'return': 16, 'error': 'configuration was not found'}

select = 0
if len(selector_with_meta) > 1:
Expand All @@ -321,16 +324,20 @@ def select_cfg(i):

print('')

selector_with_meta = sorted(selector_with_meta, key = lambda x: x['meta'].get('name', ''))
selector_with_meta = sorted(
selector_with_meta,
key=lambda x: x['meta'].get(
'name',
''))
s = 0
for ss in selector_with_meta:
alias = ss['alias']
uid = ss['meta'].get('uid', '')
name = ss['meta'].get('name', '')

x = name
if x!='':
x+=' '
if x != '':
x += ' '
x += '(' + uid + ')'

print(f'{s}) {x}'.format(s, x))
Expand All @@ -345,9 +352,9 @@ def select_cfg(i):

select = int(select)

if select <0 or select>=len(selector):
return {'return': 1, 'error':'selection is out of range'}
if select < 0 or select >= len(selector):
return {'return': 1, 'error': 'selection is out of range'}

ss = selector_with_meta[select]

return {'return': 0, 'selection':ss}
return {'return': 0, 'selection': ss}

0 comments on commit 3d2520a

Please sign in to comment.