Merge pull request #535 from mlcommons/issue-#403

enable user to submit a result for both closed and open
mlcommons · Nov 19, 2024 · 0b58485 · 0b58485
2 parents b32ded2 + d785cc0
commit 0b58485
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 17 deletions.
diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml
@@ -17,14 +17,13 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
         python-version: [ "3.12" ]
-        division: ["closed", "open"]
+        division: ["closed", "open", "closed-open"]
         category: ["datacenter", "edge"]
         case: ["case-3", "case-7", "case-8"]
         action: ["run", "docker"]
         exclude:
           - os: macos-latest
           - os: windows-latest
-          - division: "open"
           - category: "edge"
     steps:
     - uses: actions/checkout@v4

diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py
@@ -42,13 +42,10 @@ def model_in_valid_models(model, mlperf_version):
     else:
         return (True, model)
 
-def generate_submission(i):
+def generate_submission(env, state, inp, submission_division):
 
     # Save current user directory
     cur_dir=os.getcwd()
-    env = i['env']
-    state = i['state']
-    inp=i['input']
 
     if env.get('CM_MLPERF_INFERENCE_RESULTS_DIR_', '') == '':
         results_dir = os.path.join(env['CM_MLPERF_INFERENCE_RESULTS_DIR'], f"{env['CM_MLPERF_RUN_STYLE']}_results")
@@ -95,20 +92,17 @@ def generate_submission(i):
     if 'CM_MLPERF_SUBMISSION_SYSTEM_TYPE' in env:
         system_meta['system_type'] = env['CM_MLPERF_SUBMISSION_SYSTEM_TYPE']
 
-    if 'CM_MLPERF_SUBMISSION_DIVISION' in env:
-        system_meta['division'] = env['CM_MLPERF_SUBMISSION_DIVISION']
+    if submission_division != "":
+        system_meta['division'] = submission_division 
+        division = submission_division
+    else:
+        division = system_meta_default['division']
 
     if 'CM_MLPERF_SUBMISSION_CATEGORY' in env:
         system_meta['system_type'] = env['CM_MLPERF_SUBMISSION_CATEGORY'].replace("-", ",")
 
     duplicate= (env.get('CM_MLPERF_DUPLICATE_SCENARIO_RESULTS', 'no') in ["yes", "True"])
 
-    if env.get('CM_MLPERF_SUBMISSION_DIVISION', '') != '':
-        division = env['CM_MLPERF_SUBMISSION_DIVISION']
-        system_meta['division'] = division
-    else:
-        division = system_meta_default['division']
-
     if division not in ['open','closed']:
         return {'return':1, 'error':'"division" must be "open" or "closed"'}
 
@@ -495,9 +489,23 @@ def generate_submission(i):
     return {'return':0}
 
 def postprocess(i):
+    env = i['env']
+    state = i['state']
+    inp=i['input']
 
-    r = generate_submission(i)
-    if r['return'] > 0:
-        return r
+    submission_divisions = [] 
+
+    if env.get('CM_MLPERF_SUBMISSION_DIVISION', '') in ["open-closed", "closed-open"]:
+        submission_divisions = ["open", "closed"]
+    elif env.get('CM_MLPERF_SUBMISSION_DIVISION', '') != '':
+        submission_divisions.append(env['CM_MLPERF_SUBMISSION_DIVISION'])
+
+    if env.get('CM_MLPERF_SUBMISSION_DIVISION', '') == '':    #if submission division is not assigned, default value would be taken in submission_generation function
+        r = generate_submission(env, state, inp, submission_division="")
+    else:
+        for submission_division in submission_divisions:
+            r = generate_submission(env, state, inp, submission_division)
+            if r['return'] > 0:
+                return r
 
     return {'return':0}