From 2ed8d60ad06f09fbac972eeb13841edf68951760 Mon Sep 17 00:00:00 2001 From: mengfeil Date: Thu, 26 Dec 2024 15:28:17 +0800 Subject: [PATCH 1/6] update weekly accuracy referenche --- .../inductor_timm_models_inference.csv | 2 +- .../inductor_timm_models_training.csv | 8 ++++---- .../inductor_torchbench_inference.csv | 12 +++++------ .../inductor_torchbench_training.csv | 20 +++++++++---------- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/ci_expected_accuracy/inductor_timm_models_inference.csv b/.github/ci_expected_accuracy/inductor_timm_models_inference.csv index ffaa26f62..66ed332bb 100644 --- a/.github/ci_expected_accuracy/inductor_timm_models_inference.csv +++ b/.github/ci_expected_accuracy/inductor_timm_models_inference.csv @@ -59,4 +59,4 @@ twins_pcpvt_base,pass,pass,pass,pass,pass visformer_small,pass,pass,pass,pass,pass vit_base_patch16_224,pass,pass,pass,pass,pass volo_d1_224,pass,pass,pass,pass,pass -xcit_large_24_p8_224,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +xcit_large_24_p8_224,pass,pass,pass,pass,pass diff --git a/.github/ci_expected_accuracy/inductor_timm_models_training.csv b/.github/ci_expected_accuracy/inductor_timm_models_training.csv index c605c9bdf..6bbb653f0 100644 --- a/.github/ci_expected_accuracy/inductor_timm_models_training.csv +++ b/.github/ci_expected_accuracy/inductor_timm_models_training.csv @@ -14,7 +14,7 @@ dla102,pass,pass,pass,pass,pass dm_nfnet_f0,pass,pass,pass,pass,pass dpn107,pass,pass,pass,pass,pass eca_botnext26ts_256,pass,pass,pass,pass,pass -eca_halonext26ts,pass,pass,fail_to_run,pass,pass +eca_halonext26ts,pass,pass,pass,pass,pass ese_vovnet19b_dw,pass,pass,pass,pass,pass fbnetc_100,pass,pass,pass,pass,pass fbnetv3_b,pass,pass,pass,pass,pass @@ -25,8 +25,8 @@ gmixer_24_224,pass,pass,pass,pass,pass gmlp_s16_224,pass,pass,pass,pass,pass hrnet_w18,pass,pass,pass,pass,pass inception_v3,pass,pass,pass,pass,pass -jx_nest_base,pass,pass,fail_accuracy,pass,pass -lcnet_050,pass,pass,fail_accuracy,pass,pass +jx_nest_base,pass,pass,pass,pass,pass +lcnet_050,pass,pass,pass,pass,pass levit_128,pass,pass,pass,pass,pass mixer_b16_224,pass,pass,pass,pass,pass mixnet_l,pass,pass,pass,pass,pass @@ -59,4 +59,4 @@ twins_pcpvt_base,pass,pass,pass,pass,pass visformer_small,pass,pass,pass,pass,pass vit_base_patch16_224,pass,pass,pass,pass,pass volo_d1_224,pass,pass,pass,pass,pass -xcit_large_24_p8_224,pass_due_to_skip,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +xcit_large_24_p8_224,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip diff --git a/.github/ci_expected_accuracy/inductor_torchbench_inference.csv b/.github/ci_expected_accuracy/inductor_torchbench_inference.csv index 4825aa41f..792bfac0b 100644 --- a/.github/ci_expected_accuracy/inductor_torchbench_inference.csv +++ b/.github/ci_expected_accuracy/inductor_torchbench_inference.csv @@ -25,9 +25,9 @@ detectron2_maskrcnn,fail_to_run,eager_fail_to_run,fail_to_run,eager_fail_to_run, detectron2_maskrcnn_r_101_c4,fail_accuracy,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy detectron2_maskrcnn_r_101_fpn,fail_accuracy,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy detectron2_maskrcnn_r_50_c4,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy -detectron2_maskrcnn_r_50_fpn,fail_accuracy,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy +detectron2_maskrcnn_r_50_fpn,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy dlrm,pass,pass,pass,pass,pass -doctr_det_predictor,pass,pass,pass,eager_fail_to_run,pass +doctr_det_predictor,pass,pass,pass,pass,pass doctr_reco_predictor,pass,pass,pass,pass,pass drq,pass,pass,pass,pass,pass fastNLP_Bert,pass,pass,pass,pass,pass @@ -68,7 +68,7 @@ nvidia_deeprecommender,pass,pass,pass,pass,pass opacus_cifar10,pass,pass,pass,pass,pass phlippe_densenet,pass,pass,pass,pass,pass phlippe_resnet,pass,pass,pass,pass,pass -pyhpc_equation_of_state,pass,pass,fail_accuracy,pass,pass +pyhpc_equation_of_state,pass,pass,pass,pass,pass pyhpc_isoneutral_mixing,pass,pass,pass,pass,pass pyhpc_turbulent_kinetic_energy,pass,pass,pass,pass,pass pytorch_CycleGAN_and_pix2pix,pass,pass,pass,pass,pass @@ -86,7 +86,7 @@ simple_gpt,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_t simple_gpt_tp_manual,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load soft_actor_critic,pass,pass,pass,pass,pass speech_transformer,pass,pass,pass,pass,pass -squeezenet1_1,pass,fail_accuracy,fail_accuracy,pass,pass +squeezenet1_1,pass,fail_accuracy,pass,pass,pass stable_diffusion_text_encoder,pass,pass,pass,pass,pass stable_diffusion_unet,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip tacotron2,pass,pass,pass,fail_to_run,fail_to_run @@ -98,8 +98,8 @@ timm_resnest,pass,pass,pass,pass,pass timm_vision_transformer,pass,pass,pass,pass,pass timm_vision_transformer_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip timm_vovnet,pass,pass,pass,pass,pass -torch_multimodal_clip,pass,pass,pass,eager_fail_to_run,eager_fail_to_run -tts_angular,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +torch_multimodal_clip,pass,pass,pass,pass,pass +tts_angular,pass,pass,pass,pass,pass vgg16,pass,pass,pass,pass,pass vision_maskrcnn,pass,pass,pass,eager_fail_to_run,eager_fail_to_run yolov3,pass,pass,pass,pass,pass diff --git a/.github/ci_expected_accuracy/inductor_torchbench_training.csv b/.github/ci_expected_accuracy/inductor_torchbench_training.csv index dc766eac0..4c1bc6aba 100644 --- a/.github/ci_expected_accuracy/inductor_torchbench_training.csv +++ b/.github/ci_expected_accuracy/inductor_torchbench_training.csv @@ -1,11 +1,11 @@ name,float32,bfloat16,float16,amp_bf16,amp_fp16 -torchrec_dlrm,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +torchrec_dlrm,pass,eager_fail_to_run,eager_fail_to_run,pass,pass BERT_pytorch,pass,pass,pass,pass,pass Background_Matting,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip DALLE2_pytorch,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run LearningToPaint,pass,pass,pass,pass,pass -Super_SloMo,eager_two_runs_differ,pass,pass,eager_two_runs_differ,pass -alexnet,eager_two_runs_differ,pass,pass,pass,pass +Super_SloMo,eager_two_runs_differ,pass,pass,pass,pass +alexnet,pass,pass,pass,pass,pass basic_gnn_edgecnn,pass,pass,pass,pass,pass basic_gnn_gcn,pass,pass,pass,pass,pass basic_gnn_gin,pass,pass,pass,pass,pass @@ -26,13 +26,13 @@ detectron2_maskrcnn_r_101_c4,eager_fail_to_run,eager_fail_to_run,eager_fail_to_r detectron2_maskrcnn_r_101_fpn,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run detectron2_maskrcnn_r_50_c4,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run detectron2_maskrcnn_r_50_fpn,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run -dlrm,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +dlrm,pass,pass,pass,pass,pass doctr_det_predictor,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run doctr_reco_predictor,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run drq,pass,pass,pass,pass,pass fastNLP_Bert,pass,pass,pass,pass,pass functorch_dp_cifar10,fail_accuracy,fail_accuracy,fail_accuracy,pass,pass -functorch_maml_omniglot,pass,pass,pass,fail_accuracy,pass +functorch_maml_omniglot,pass,pass,pass,eager_fail_to_run,pass hf_Albert,pass,pass,pass,pass,pass hf_Bart,pass,pass,pass,pass,pass hf_Bert,pass,pass,pass,pass,pass @@ -59,7 +59,7 @@ maml_omniglot,pass,pass,pass,pass,pass microbench_unbacked_tolist_sum,pass,pass,pass,pass,pass mnasnet1_0,pass,pass,pass,pass,pass mobilenet_v2,pass,pass,pass,pass,pass -mobilenet_v2_quantized_qat,fail_accuracy,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +mobilenet_v2_quantized_qat,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run mobilenet_v3_large,pass,pass,pass,pass,pass moco,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load moondream,pass,pass,pass,pass,pass @@ -77,7 +77,7 @@ pytorch_unet,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip resnet152,pass,pass,pass,pass,pass resnet18,pass,pass,pass,pass,pass resnet50,pass,pass,pass,pass,pass -resnet50_quantized_qat,eager_two_runs_differ,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +resnet50_quantized_qat,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run resnext50_32x4d,pass,pass,pass,pass,pass sam,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run sam_fast,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run @@ -86,7 +86,7 @@ simple_gpt,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_t simple_gpt_tp_manual,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load soft_actor_critic,pass,pass,pass,pass,pass speech_transformer,pass,pass,pass,pass,pass -squeezenet1_1,pass,fail_accuracy,fail_accuracy,fail_accuracy,pass +squeezenet1_1,pass,pass,pass,pass,pass stable_diffusion_text_encoder,pass,pass,pass,pass,pass stable_diffusion_unet,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip tacotron2,fail_to_run,fail_to_run,fail_to_run,fail_to_run,fail_to_run @@ -98,8 +98,8 @@ timm_resnest,pass,pass,pass,pass,pass timm_vision_transformer,pass,pass,pass,pass,pass timm_vision_transformer_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip timm_vovnet,pass,pass,pass,pass,pass -torch_multimodal_clip,pass,pass,pass,eager_fail_to_run,eager_fail_to_run -tts_angular,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +torch_multimodal_clip,pass,pass,pass,pass,pass +tts_angular,pass,pass,pass,pass,pass vgg16,pass,pass,pass,pass,pass vision_maskrcnn,pass,pass,pass,eager_fail_to_run,eager_fail_to_run yolov3,pass,pass,pass,pass,pass From a905468d9cdb226af1ad1967698d174db5180e44 Mon Sep 17 00:00:00 2001 From: mengfeil Date: Thu, 26 Dec 2024 15:56:45 +0800 Subject: [PATCH 2/6] Update failed models with known issues --- .../inductor_huggingface_training.csv | 2 +- .../inductor_timm_models_inference.csv | 40 ++++++------ .../inductor_timm_models_training.csv | 62 +++++++++---------- .../inductor_torchbench_inference.csv | 38 ++++++------ .../inductor_torchbench_training.csv | 34 +++++----- 5 files changed, 88 insertions(+), 88 deletions(-) diff --git a/.github/ci_expected_accuracy/inductor_huggingface_training.csv b/.github/ci_expected_accuracy/inductor_huggingface_training.csv index a75d3d225..fa73dc6fc 100644 --- a/.github/ci_expected_accuracy/inductor_huggingface_training.csv +++ b/.github/ci_expected_accuracy/inductor_huggingface_training.csv @@ -13,7 +13,7 @@ CamemBert,pass,pass,pass,pass,pass DebertaForMaskedLM,pass,pass,pass,pass,pass DebertaForQuestionAnswering,pass,pass,pass,pass,pass DebertaV2ForMaskedLM,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip -DebertaV2ForQuestionAnswering,pass,pass,pass,pass,pass +DebertaV2ForQuestionAnswering,fail_accuracy,fail_accuracy,fail_accuracy,pass,pass DistilBertForMaskedLM,pass,pass,pass,pass,pass DistilBertForQuestionAnswering,pass,pass,pass,pass,pass DistillGPT2,pass,pass,pass,pass,pass diff --git a/.github/ci_expected_accuracy/inductor_timm_models_inference.csv b/.github/ci_expected_accuracy/inductor_timm_models_inference.csv index 66ed332bb..42e454c6c 100644 --- a/.github/ci_expected_accuracy/inductor_timm_models_inference.csv +++ b/.github/ci_expected_accuracy/inductor_timm_models_inference.csv @@ -1,5 +1,5 @@ name,float32,bfloat16,float16,amp_bf16,amp_fp16 -adv_inception_v3,pass,pass,pass,pass,pass +adv_inception_v3,pass,pass,pass,fail_accuracy,pass beit_base_patch16_224,pass,pass,pass,pass,pass botnet26t_256,pass,pass,pass,pass,pass cait_m36_384,pass,pass,pass,pass,pass @@ -10,48 +10,48 @@ convnext_base,pass,pass,pass,pass,pass crossvit_9_240,pass,pass,pass,pass,pass cspdarknet53,pass,pass,pass,pass,pass deit_base_distilled_patch16_224,pass,pass,pass,pass,pass -dla102,pass,pass,pass,pass,pass -dm_nfnet_f0,pass,pass,pass,pass,pass -dpn107,pass,pass,pass,pass,pass +dla102,pass,fail_accuracy,pass,fail_accuracy,pass +dm_nfnet_f0,pass,fail_accuracy,pass,pass,pass +dpn107,pass,fail_accuracy,pass,fail_accuracy,pass eca_botnext26ts_256,pass,pass,pass,pass,pass eca_halonext26ts,pass,pass,pass,pass,pass ese_vovnet19b_dw,pass,pass,pass,pass,pass -fbnetc_100,pass,pass,pass,pass,pass +fbnetc_100,pass,fail_accuracy,pass,fail_accuracy,pass fbnetv3_b,pass,pass,pass,pass,pass gernet_l,pass,pass,pass,pass,pass -ghostnet_100,pass,pass,pass,pass,pass -gluon_inception_v3,pass,pass,pass,pass,pass +ghostnet_100,pass,fail_accuracy,pass,fail_accuracy,pass +gluon_inception_v3,pass,pass,pass,fail_accuracy,pass gmixer_24_224,pass,pass,pass,pass,pass gmlp_s16_224,pass,pass,pass,pass,pass -hrnet_w18,pass,pass,pass,pass,pass -inception_v3,pass,pass,pass,pass,pass +hrnet_w18,pass,fail_accuracy,pass,fail_accuracy,pass +inception_v3,pass,fail_accuracy,pass,fail_accuracy,pass jx_nest_base,pass,pass,pass,pass,pass -lcnet_050,pass,pass,pass,pass,pass +lcnet_050,pass,fail_accuracy,pass,fail_accuracy,pass levit_128,pass,pass,pass,pass,pass mixer_b16_224,pass,pass,pass,pass,pass mixnet_l,pass,pass,pass,pass,pass -mnasnet_100,pass,pass,pass,pass,pass -mobilenetv2_100,pass,pass,pass,pass,pass +mnasnet_100,pass,fail_accuracy,pass,fail_accuracy,pass +mobilenetv2_100,pass,pass,pass,fail_accuracy,pass mobilenetv3_large_100,pass,pass,pass,pass,pass mobilevit_s,pass,pass,pass,pass,pass nfnet_l0,pass,pass,pass,pass,pass pit_b_224,pass,pass,pass,pass,pass pnasnet5large,pass,pass,pass,pass,pass poolformer_m36,pass,pass,pass,pass,pass -regnety_002,pass,pass,pass,pass,pass +regnety_002,pass,fail_accuracy,pass,fail_accuracy,pass repvgg_a2,pass,pass,pass,pass,pass -res2net101_26w_4s,pass,pass,pass,pass,pass -res2net50_14w_8s,pass,pass,pass,pass,pass -res2next50,pass,pass,pass,pass,pass +res2net101_26w_4s,pass,pass,pass,fail_accuracy,pass +res2net50_14w_8s,pass,pass,pass,fail_accuracy,pass +res2next50,pass,pass,pass,fail_accuracy,pass resmlp_12_224,pass,pass,pass,pass,pass resnest101e,pass,pass,pass,pass,pass -rexnet_100,pass,pass,pass,pass,pass +rexnet_100,pass,pass,pass,fail_accuracy,pass sebotnet33ts_256,pass,pass,pass,pass,pass selecsls42b,pass,pass,pass,pass,pass -spnasnet_100,pass,pass,pass,pass,pass +spnasnet_100,pass,fail_accuracy,pass,fail_accuracy,pass swin_base_patch4_window7_224,pass,pass,pass,pass,pass -swsl_resnext101_32x16d,pass,pass,pass,pass,pass -tf_efficientnet_b0,pass,pass,pass,pass,pass +swsl_resnext101_32x16d,pass,fail_accuracy,pass,fail_accuracy,pass +tf_efficientnet_b0,pass,fail_accuracy,pass,fail_accuracy,pass tf_mixnet_l,pass,pass,pass,pass,pass tinynet_a,pass,pass,pass,pass,pass tnt_s_patch16_224,pass,pass,pass,pass,pass diff --git a/.github/ci_expected_accuracy/inductor_timm_models_training.csv b/.github/ci_expected_accuracy/inductor_timm_models_training.csv index 6bbb653f0..6da0ad06f 100644 --- a/.github/ci_expected_accuracy/inductor_timm_models_training.csv +++ b/.github/ci_expected_accuracy/inductor_timm_models_training.csv @@ -1,62 +1,62 @@ name,float32,bfloat16,float16,amp_bf16,amp_fp16 -adv_inception_v3,pass,pass,pass,pass,pass +adv_inception_v3,pass,fail_accuracy,pass,fail_accuracy,pass beit_base_patch16_224,pass,pass,pass,pass,pass botnet26t_256,pass,pass,pass,pass,pass cait_m36_384,pass,pass,pass,pass,pass coat_lite_mini,pass,pass,pass,pass,pass convit_base,pass,pass,pass,pass,pass -convmixer_768_32,pass,pass,pass,pass,pass -convnext_base,pass,pass,fail_accuracy,pass,pass +convmixer_768_32,pass,fail_accuracy,pass,fail_accuracy,pass +convnext_base,pass,fail_accuracy,fail_accuracy,pass,pass crossvit_9_240,pass,pass,pass,pass,pass cspdarknet53,pass,pass,pass,pass,pass deit_base_distilled_patch16_224,pass,pass,pass,pass,pass -dla102,pass,pass,pass,pass,pass -dm_nfnet_f0,pass,pass,pass,pass,pass +dla102,pass,fail_accuracy,pass,fail_accuracy,pass +dm_nfnet_f0,pass,fail_accuracy,pass,pass,pass dpn107,pass,pass,pass,pass,pass eca_botnext26ts_256,pass,pass,pass,pass,pass -eca_halonext26ts,pass,pass,pass,pass,pass -ese_vovnet19b_dw,pass,pass,pass,pass,pass -fbnetc_100,pass,pass,pass,pass,pass -fbnetv3_b,pass,pass,pass,pass,pass +eca_halonext26ts,pass,pass,pass,fail_accuracy,pass +ese_vovnet19b_dw,pass,fail_accuracy,pass,fail_accuracy,pass +fbnetc_100,pass,fail_accuracy,pass,fail_accuracy,pass +fbnetv3_b,pass,fail_accuracy,pass,fail_accuracy,pass gernet_l,pass,pass,pass,pass,pass -ghostnet_100,pass,pass,pass,pass,pass -gluon_inception_v3,pass,pass,pass,pass,pass +ghostnet_100,pass,fail_accuracy,pass,fail_accuracy,pass +gluon_inception_v3,pass,fail_accuracy,pass,fail_accuracy,pass gmixer_24_224,pass,pass,pass,pass,pass gmlp_s16_224,pass,pass,pass,pass,pass -hrnet_w18,pass,pass,pass,pass,pass -inception_v3,pass,pass,pass,pass,pass +hrnet_w18,pass,fail_accuracy,pass,fail_accuracy,pass +inception_v3,pass,fail_accuracy,pass,fail_accuracy,pass jx_nest_base,pass,pass,pass,pass,pass -lcnet_050,pass,pass,pass,pass,pass +lcnet_050,pass,fail_accuracy,pass,fail_accuracy,pass levit_128,pass,pass,pass,pass,pass mixer_b16_224,pass,pass,pass,pass,pass -mixnet_l,pass,pass,pass,pass,pass -mnasnet_100,pass,pass,pass,pass,pass -mobilenetv2_100,pass,pass,pass,pass,pass -mobilenetv3_large_100,pass,pass,pass,pass,pass +mixnet_l,pass,fail_accuracy,pass,fail_accuracy,pass +mnasnet_100,pass,fail_accuracy,pass,fail_accuracy,pass +mobilenetv2_100,pass,fail_accuracy,pass,fail_accuracy,pass +mobilenetv3_large_100,pass,fail_accuracy,pass,fail_accuracy,pass mobilevit_s,pass,pass,pass,pass,pass nfnet_l0,pass,pass,pass,pass,pass pit_b_224,pass,pass,pass,pass,pass -pnasnet5large,pass,pass,pass,pass,pass +pnasnet5large,pass,fail_accuracy,pass,fail_accuracy,pass poolformer_m36,pass,pass,pass,pass,pass -regnety_002,pass,pass,pass,pass,pass -repvgg_a2,pass,pass,pass,pass,pass -res2net101_26w_4s,pass,pass,pass,pass,pass -res2net50_14w_8s,pass,pass,pass,pass,pass -res2next50,pass,pass,pass,pass,pass +regnety_002,pass,fail_accuracy,pass,fail_accuracy,pass +repvgg_a2,pass,fail_accuracy,pass,fail_accuracy,pass +res2net101_26w_4s,pass,fail_accuracy,pass,fail_accuracy,pass +res2net50_14w_8s,pass,fail_accuracy,pass,fail_accuracy,pass +res2next50,pass,fail_accuracy,pass,fail_accuracy,pass resmlp_12_224,pass,pass,pass,pass,pass resnest101e,pass,pass,pass,pass,pass -rexnet_100,pass,pass,pass,pass,pass +rexnet_100,pass,fail_accuracy,pass,fail_accuracy,pass sebotnet33ts_256,pass,pass,pass,pass,pass -selecsls42b,pass,pass,pass,pass,pass -spnasnet_100,pass,pass,pass,pass,pass +selecsls42b,pass,fail_accuracy,pass,fail_accuracy,pass +spnasnet_100,pass,fail_accuracy,pass,fail_accuracy,pass swin_base_patch4_window7_224,pass,pass,pass,pass,pass -swsl_resnext101_32x16d,pass,pass,pass,pass,pass -tf_efficientnet_b0,pass,pass,pass,pass,pass -tf_mixnet_l,pass,pass,pass,pass,pass +swsl_resnext101_32x16d,pass,fail_accuracy,pass,fail_accuracy,pass +tf_efficientnet_b0,pass,fail_accuracy,pass,fail_accuracy,pass +tf_mixnet_l,pass,fail_accuracy,pass,fail_accuracy,pass tinynet_a,pass,pass,pass,pass,pass tnt_s_patch16_224,pass,pass,pass,pass,pass twins_pcpvt_base,pass,pass,pass,pass,pass -visformer_small,pass,pass,pass,pass,pass +visformer_small,pass,fail_accuracy,pass,fail_accuracy,pass vit_base_patch16_224,pass,pass,pass,pass,pass volo_d1_224,pass,pass,pass,pass,pass xcit_large_24_p8_224,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip diff --git a/.github/ci_expected_accuracy/inductor_torchbench_inference.csv b/.github/ci_expected_accuracy/inductor_torchbench_inference.csv index 792bfac0b..f26592d43 100644 --- a/.github/ci_expected_accuracy/inductor_torchbench_inference.csv +++ b/.github/ci_expected_accuracy/inductor_torchbench_inference.csv @@ -1,5 +1,5 @@ name,float32,bfloat16,float16,amp_bf16,amp_fp16 -torchrec_dlrm,pass,eager_fail_to_run,eager_fail_to_run,pass,pass +torchrec_dlrm,pass,eager_fail_to_run,eager_fail_to_run,fail_to_run,fail_to_run BERT_pytorch,pass,pass,pass,pass,pass Background_Matting,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip DALLE2_pytorch,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run @@ -13,7 +13,7 @@ basic_gnn_sage,pass,pass,pass,pass,pass cm3leon_generate,pass,pass,pass,pass,pass dcgan,pass,pass,pass,pass,pass demucs,pass,pass,pass,pass,pass -densenet121,pass,pass,pass,pass,pass +densenet121,pass,pass,pass,fail_accuracy,pass detectron2_fasterrcnn_r_101_c4,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy detectron2_fasterrcnn_r_101_dc5,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy detectron2_fasterrcnn_r_101_fpn,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy @@ -32,7 +32,7 @@ doctr_reco_predictor,pass,pass,pass,pass,pass drq,pass,pass,pass,pass,pass fastNLP_Bert,pass,pass,pass,pass,pass functorch_dp_cifar10,pass,pass,pass,pass,pass -functorch_maml_omniglot,pass,pass,pass,pass,pass +functorch_maml_omniglot,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run hf_Albert,pass,pass,pass,pass,pass hf_Bart,pass,pass,pass,pass,pass hf_Bert,pass,pass,pass,pass,pass @@ -44,7 +44,7 @@ hf_GPT2_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_ski hf_Longformer,pass,pass,pass,pass,pass hf_Reformer,pass,pass,pass,pass,pass hf_T5,pass,pass,pass,pass,pass -hf_T5_base,pass,pass,pass,pass,pass +hf_T5_base,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run hf_T5_generate,pass,pass,pass,pass,pass hf_T5_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip hf_Whisper,pass,pass,pass,pass,pass @@ -53,19 +53,19 @@ hf_distil_whisper,pass,pass,pass,pass,pass lennard_jones,pass,pass,pass,pass,pass llama,pass,pass,pass,pass,pass llama_v2_7b_16h,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip -llava,eager_fail_to_run,pass,pass,eager_fail_to_run,eager_fail_to_run +llava,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run maml,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip -maml_omniglot,pass,pass,pass,pass,pass +maml_omniglot,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run microbench_unbacked_tolist_sum,pass,pass,pass,pass,pass -mnasnet1_0,pass,pass,pass,pass,pass -mobilenet_v2,pass,pass,pass,pass,pass +mnasnet1_0,pass,fail_accuracy,pass,fail_accuracy,pass +mobilenet_v2,pass,fail_accuracy,pass,fail_accuracy,pass mobilenet_v2_quantized_qat,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load mobilenet_v3_large,pass,pass,pass,pass,pass moco,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load moondream,pass,pass,pass,pass,pass nanogpt,pass,pass,pass,pass,pass nvidia_deeprecommender,pass,pass,pass,pass,pass -opacus_cifar10,pass,pass,pass,pass,pass +opacus_cifar10,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run phlippe_densenet,pass,pass,pass,pass,pass phlippe_resnet,pass,pass,pass,pass,pass pyhpc_equation_of_state,pass,pass,pass,pass,pass @@ -74,30 +74,30 @@ pyhpc_turbulent_kinetic_energy,pass,pass,pass,pass,pass pytorch_CycleGAN_and_pix2pix,pass,pass,pass,pass,pass pytorch_stargan,pass,pass,pass,pass,pass pytorch_unet,pass,pass,pass,pass,pass -resnet152,pass,pass,pass,pass,pass -resnet18,pass,pass,pass,pass,pass +resnet152,pass,fail_accuracy,pass,fail_accuracy,pass +resnet18,pass,pass,pass,fail_accuracy,pass resnet50,pass,pass,pass,pass,pass resnet50_quantized_qat,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load -resnext50_32x4d,pass,pass,pass,pass,pass +resnext50_32x4d,pass,pass,pass,fail_accuracy,pass sam,pass,pass,pass,pass,pass sam_fast,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run -shufflenet_v2_x1_0,pass,pass,pass,pass,pass +shufflenet_v2_x1_0,pass,fail_accuracy,pass,fail_accuracy,pass simple_gpt,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load simple_gpt_tp_manual,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load soft_actor_critic,pass,pass,pass,pass,pass speech_transformer,pass,pass,pass,pass,pass squeezenet1_1,pass,fail_accuracy,pass,pass,pass -stable_diffusion_text_encoder,pass,pass,pass,pass,pass -stable_diffusion_unet,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip +stable_diffusion_text_encoder,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +stable_diffusion_unet,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run tacotron2,pass,pass,pass,fail_to_run,fail_to_run timm_efficientdet,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load timm_efficientnet,pass,pass,pass,pass,pass -timm_nfnet,pass,pass,pass,pass,pass -timm_regnet,pass,pass,pass,pass,pass -timm_resnest,pass,pass,pass,pass,pass +timm_nfnet,pass,fail_accuracy,pass,pass,pass +timm_regnet,pass,pass,pass,fail_accuracy,pass +timm_resnest,pass,fail_accuracy,pass,fail_accuracy,pass timm_vision_transformer,pass,pass,pass,pass,pass timm_vision_transformer_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip -timm_vovnet,pass,pass,pass,pass,pass +timm_vovnet,pass,fail_accuracy,pass,fail_accuracy,pass torch_multimodal_clip,pass,pass,pass,pass,pass tts_angular,pass,pass,pass,pass,pass vgg16,pass,pass,pass,pass,pass diff --git a/.github/ci_expected_accuracy/inductor_torchbench_training.csv b/.github/ci_expected_accuracy/inductor_torchbench_training.csv index 4c1bc6aba..4b4322c88 100644 --- a/.github/ci_expected_accuracy/inductor_torchbench_training.csv +++ b/.github/ci_expected_accuracy/inductor_torchbench_training.csv @@ -13,7 +13,7 @@ basic_gnn_sage,pass,pass,pass,pass,pass cm3leon_generate,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run dcgan,pass,pass,pass,pass,pass demucs,fail_accuracy,eager_fail_to_run,eager_fail_to_run,fail_accuracy,fail_accuracy -densenet121,pass,pass,pass,pass,pass +densenet121,pass,pass,pass,fail_accuracy,pass detectron2_fasterrcnn_r_101_c4,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run detectron2_fasterrcnn_r_101_dc5,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run detectron2_fasterrcnn_r_101_fpn,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run @@ -32,7 +32,7 @@ doctr_reco_predictor,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager drq,pass,pass,pass,pass,pass fastNLP_Bert,pass,pass,pass,pass,pass functorch_dp_cifar10,fail_accuracy,fail_accuracy,fail_accuracy,pass,pass -functorch_maml_omniglot,pass,pass,pass,eager_fail_to_run,pass +functorch_maml_omniglot,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run hf_Albert,pass,pass,pass,pass,pass hf_Bart,pass,pass,pass,pass,pass hf_Bert,pass,pass,pass,pass,pass @@ -55,10 +55,10 @@ llama,pass,pass,pass,pass,pass llama_v2_7b_16h,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip llava,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run maml,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run -maml_omniglot,pass,pass,pass,pass,pass +maml_omniglot,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run microbench_unbacked_tolist_sum,pass,pass,pass,pass,pass -mnasnet1_0,pass,pass,pass,pass,pass -mobilenet_v2,pass,pass,pass,pass,pass +mnasnet1_0,pass,fail_accuracy,pass,fail_accuracy,pass +mobilenet_v2,pass,fail_accuracy,pass,fail_accuracy,pass mobilenet_v2_quantized_qat,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run mobilenet_v3_large,pass,pass,pass,pass,pass moco,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load @@ -74,33 +74,33 @@ pyhpc_turbulent_kinetic_energy,model_fail_to_load,model_fail_to_load,model_fail_ pytorch_CycleGAN_and_pix2pix,eager_two_runs_differ,eager_two_runs_differ,pass,eager_two_runs_differ,pass pytorch_stargan,pass,pass,pass,pass,pass pytorch_unet,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip -resnet152,pass,pass,pass,pass,pass -resnet18,pass,pass,pass,pass,pass -resnet50,pass,pass,pass,pass,pass +resnet152,pass,fail_accuracy,pass,fail_accuracy,pass +resnet18,pass,pass,pass,fail_accuracy,pass +resnet50,pass,pass,pass,fail_accuracy,pass resnet50_quantized_qat,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run -resnext50_32x4d,pass,pass,pass,pass,pass +resnext50_32x4d,pass,pass,pass,fail_accuracy,pass sam,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run sam_fast,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run -shufflenet_v2_x1_0,pass,pass,pass,pass,pass +shufflenet_v2_x1_0,pass,fail_accuracy,pass,fail_accuracy,pass simple_gpt,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load simple_gpt_tp_manual,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load soft_actor_critic,pass,pass,pass,pass,pass speech_transformer,pass,pass,pass,pass,pass squeezenet1_1,pass,pass,pass,pass,pass -stable_diffusion_text_encoder,pass,pass,pass,pass,pass -stable_diffusion_unet,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip +stable_diffusion_text_encoder,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +stable_diffusion_unet,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run tacotron2,fail_to_run,fail_to_run,fail_to_run,fail_to_run,fail_to_run timm_efficientdet,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load timm_efficientnet,pass,pass,pass,pass,pass -timm_nfnet,pass,pass,pass,pass,pass -timm_regnet,pass,fail_accuracy,pass,pass,pass -timm_resnest,pass,pass,pass,pass,pass +timm_nfnet,pass,fail_accuracy,pass,pass,pass +timm_regnet,pass,fail_accuracy,fail_accuracy,fail_accuracy,fail_accuracy +timm_resnest,pass,fail_accuracy,pass,fail_accuracy,pass timm_vision_transformer,pass,pass,pass,pass,pass timm_vision_transformer_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip -timm_vovnet,pass,pass,pass,pass,pass +timm_vovnet,pass,fail_accuracy,pass,fail_accuracy,pass torch_multimodal_clip,pass,pass,pass,pass,pass tts_angular,pass,pass,pass,pass,pass vgg16,pass,pass,pass,pass,pass -vision_maskrcnn,pass,pass,pass,eager_fail_to_run,eager_fail_to_run +vision_maskrcnn,eager_fail_to_run,pass,pass,eager_fail_to_run,eager_fail_to_run yolov3,pass,pass,pass,pass,pass hf_Roberta_base,pass,pass,pass,pass,pass From c9426d44c215b4cc29915239f8dedf9808642888 Mon Sep 17 00:00:00 2001 From: mengfeil Date: Wed, 8 Jan 2025 16:03:56 +0800 Subject: [PATCH 3/6] update 20250104 new pass --- .../inductor_timm_models_inference.csv | 40 ++++++------- .../inductor_timm_models_training.csv | 58 +++++++++---------- .../inductor_torchbench_inference.csv | 37 ++++++------ .../inductor_torchbench_training.csv | 35 ++++++----- 4 files changed, 84 insertions(+), 86 deletions(-) diff --git a/.github/ci_expected_accuracy/inductor_timm_models_inference.csv b/.github/ci_expected_accuracy/inductor_timm_models_inference.csv index 42e454c6c..66ed332bb 100644 --- a/.github/ci_expected_accuracy/inductor_timm_models_inference.csv +++ b/.github/ci_expected_accuracy/inductor_timm_models_inference.csv @@ -1,5 +1,5 @@ name,float32,bfloat16,float16,amp_bf16,amp_fp16 -adv_inception_v3,pass,pass,pass,fail_accuracy,pass +adv_inception_v3,pass,pass,pass,pass,pass beit_base_patch16_224,pass,pass,pass,pass,pass botnet26t_256,pass,pass,pass,pass,pass cait_m36_384,pass,pass,pass,pass,pass @@ -10,48 +10,48 @@ convnext_base,pass,pass,pass,pass,pass crossvit_9_240,pass,pass,pass,pass,pass cspdarknet53,pass,pass,pass,pass,pass deit_base_distilled_patch16_224,pass,pass,pass,pass,pass -dla102,pass,fail_accuracy,pass,fail_accuracy,pass -dm_nfnet_f0,pass,fail_accuracy,pass,pass,pass -dpn107,pass,fail_accuracy,pass,fail_accuracy,pass +dla102,pass,pass,pass,pass,pass +dm_nfnet_f0,pass,pass,pass,pass,pass +dpn107,pass,pass,pass,pass,pass eca_botnext26ts_256,pass,pass,pass,pass,pass eca_halonext26ts,pass,pass,pass,pass,pass ese_vovnet19b_dw,pass,pass,pass,pass,pass -fbnetc_100,pass,fail_accuracy,pass,fail_accuracy,pass +fbnetc_100,pass,pass,pass,pass,pass fbnetv3_b,pass,pass,pass,pass,pass gernet_l,pass,pass,pass,pass,pass -ghostnet_100,pass,fail_accuracy,pass,fail_accuracy,pass -gluon_inception_v3,pass,pass,pass,fail_accuracy,pass +ghostnet_100,pass,pass,pass,pass,pass +gluon_inception_v3,pass,pass,pass,pass,pass gmixer_24_224,pass,pass,pass,pass,pass gmlp_s16_224,pass,pass,pass,pass,pass -hrnet_w18,pass,fail_accuracy,pass,fail_accuracy,pass -inception_v3,pass,fail_accuracy,pass,fail_accuracy,pass +hrnet_w18,pass,pass,pass,pass,pass +inception_v3,pass,pass,pass,pass,pass jx_nest_base,pass,pass,pass,pass,pass -lcnet_050,pass,fail_accuracy,pass,fail_accuracy,pass +lcnet_050,pass,pass,pass,pass,pass levit_128,pass,pass,pass,pass,pass mixer_b16_224,pass,pass,pass,pass,pass mixnet_l,pass,pass,pass,pass,pass -mnasnet_100,pass,fail_accuracy,pass,fail_accuracy,pass -mobilenetv2_100,pass,pass,pass,fail_accuracy,pass +mnasnet_100,pass,pass,pass,pass,pass +mobilenetv2_100,pass,pass,pass,pass,pass mobilenetv3_large_100,pass,pass,pass,pass,pass mobilevit_s,pass,pass,pass,pass,pass nfnet_l0,pass,pass,pass,pass,pass pit_b_224,pass,pass,pass,pass,pass pnasnet5large,pass,pass,pass,pass,pass poolformer_m36,pass,pass,pass,pass,pass -regnety_002,pass,fail_accuracy,pass,fail_accuracy,pass +regnety_002,pass,pass,pass,pass,pass repvgg_a2,pass,pass,pass,pass,pass -res2net101_26w_4s,pass,pass,pass,fail_accuracy,pass -res2net50_14w_8s,pass,pass,pass,fail_accuracy,pass -res2next50,pass,pass,pass,fail_accuracy,pass +res2net101_26w_4s,pass,pass,pass,pass,pass +res2net50_14w_8s,pass,pass,pass,pass,pass +res2next50,pass,pass,pass,pass,pass resmlp_12_224,pass,pass,pass,pass,pass resnest101e,pass,pass,pass,pass,pass -rexnet_100,pass,pass,pass,fail_accuracy,pass +rexnet_100,pass,pass,pass,pass,pass sebotnet33ts_256,pass,pass,pass,pass,pass selecsls42b,pass,pass,pass,pass,pass -spnasnet_100,pass,fail_accuracy,pass,fail_accuracy,pass +spnasnet_100,pass,pass,pass,pass,pass swin_base_patch4_window7_224,pass,pass,pass,pass,pass -swsl_resnext101_32x16d,pass,fail_accuracy,pass,fail_accuracy,pass -tf_efficientnet_b0,pass,fail_accuracy,pass,fail_accuracy,pass +swsl_resnext101_32x16d,pass,pass,pass,pass,pass +tf_efficientnet_b0,pass,pass,pass,pass,pass tf_mixnet_l,pass,pass,pass,pass,pass tinynet_a,pass,pass,pass,pass,pass tnt_s_patch16_224,pass,pass,pass,pass,pass diff --git a/.github/ci_expected_accuracy/inductor_timm_models_training.csv b/.github/ci_expected_accuracy/inductor_timm_models_training.csv index 6da0ad06f..e723a49d3 100644 --- a/.github/ci_expected_accuracy/inductor_timm_models_training.csv +++ b/.github/ci_expected_accuracy/inductor_timm_models_training.csv @@ -1,62 +1,62 @@ name,float32,bfloat16,float16,amp_bf16,amp_fp16 -adv_inception_v3,pass,fail_accuracy,pass,fail_accuracy,pass +adv_inception_v3,pass,pass,pass,pass,pass beit_base_patch16_224,pass,pass,pass,pass,pass botnet26t_256,pass,pass,pass,pass,pass cait_m36_384,pass,pass,pass,pass,pass coat_lite_mini,pass,pass,pass,pass,pass convit_base,pass,pass,pass,pass,pass -convmixer_768_32,pass,fail_accuracy,pass,fail_accuracy,pass +convmixer_768_32,pass,pass,pass,pass,pass convnext_base,pass,fail_accuracy,fail_accuracy,pass,pass crossvit_9_240,pass,pass,pass,pass,pass cspdarknet53,pass,pass,pass,pass,pass deit_base_distilled_patch16_224,pass,pass,pass,pass,pass -dla102,pass,fail_accuracy,pass,fail_accuracy,pass -dm_nfnet_f0,pass,fail_accuracy,pass,pass,pass +dla102,pass,pass,pass,pass,pass +dm_nfnet_f0,pass,pass,pass,pass,pass dpn107,pass,pass,pass,pass,pass eca_botnext26ts_256,pass,pass,pass,pass,pass eca_halonext26ts,pass,pass,pass,fail_accuracy,pass -ese_vovnet19b_dw,pass,fail_accuracy,pass,fail_accuracy,pass -fbnetc_100,pass,fail_accuracy,pass,fail_accuracy,pass -fbnetv3_b,pass,fail_accuracy,pass,fail_accuracy,pass +ese_vovnet19b_dw,pass,pass,pass,pass,pass +fbnetc_100,pass,pass,pass,pass,pass +fbnetv3_b,pass,pass,pass,fail_accuracy,pass gernet_l,pass,pass,pass,pass,pass -ghostnet_100,pass,fail_accuracy,pass,fail_accuracy,pass -gluon_inception_v3,pass,fail_accuracy,pass,fail_accuracy,pass +ghostnet_100,pass,pass,pass,pass,pass +gluon_inception_v3,pass,pass,pass,pass,pass gmixer_24_224,pass,pass,pass,pass,pass gmlp_s16_224,pass,pass,pass,pass,pass -hrnet_w18,pass,fail_accuracy,pass,fail_accuracy,pass -inception_v3,pass,fail_accuracy,pass,fail_accuracy,pass +hrnet_w18,pass,pass,pass,pass,pass +inception_v3,pass,pass,pass,pass,pass jx_nest_base,pass,pass,pass,pass,pass -lcnet_050,pass,fail_accuracy,pass,fail_accuracy,pass +lcnet_050,pass,pass,pass,pass,pass levit_128,pass,pass,pass,pass,pass mixer_b16_224,pass,pass,pass,pass,pass -mixnet_l,pass,fail_accuracy,pass,fail_accuracy,pass -mnasnet_100,pass,fail_accuracy,pass,fail_accuracy,pass -mobilenetv2_100,pass,fail_accuracy,pass,fail_accuracy,pass -mobilenetv3_large_100,pass,fail_accuracy,pass,fail_accuracy,pass +mixnet_l,pass,pass,pass,pass,pass +mnasnet_100,pass,pass,pass,pass,pass +mobilenetv2_100,pass,pass,pass,pass,pass +mobilenetv3_large_100,pass,pass,pass,pass,pass mobilevit_s,pass,pass,pass,pass,pass nfnet_l0,pass,pass,pass,pass,pass pit_b_224,pass,pass,pass,pass,pass -pnasnet5large,pass,fail_accuracy,pass,fail_accuracy,pass +pnasnet5large,pass,pass,pass,pass,pass poolformer_m36,pass,pass,pass,pass,pass -regnety_002,pass,fail_accuracy,pass,fail_accuracy,pass -repvgg_a2,pass,fail_accuracy,pass,fail_accuracy,pass -res2net101_26w_4s,pass,fail_accuracy,pass,fail_accuracy,pass -res2net50_14w_8s,pass,fail_accuracy,pass,fail_accuracy,pass -res2next50,pass,fail_accuracy,pass,fail_accuracy,pass +regnety_002,pass,pass,pass,pass,pass +repvgg_a2,pass,pass,pass,pass,pass +res2net101_26w_4s,pass,pass,pass,pass,pass +res2net50_14w_8s,pass,pass,pass,pass,pass +res2next50,pass,pass,pass,pass,pass resmlp_12_224,pass,pass,pass,pass,pass resnest101e,pass,pass,pass,pass,pass -rexnet_100,pass,fail_accuracy,pass,fail_accuracy,pass +rexnet_100,pass,pass,pass,pass,pass sebotnet33ts_256,pass,pass,pass,pass,pass -selecsls42b,pass,fail_accuracy,pass,fail_accuracy,pass -spnasnet_100,pass,fail_accuracy,pass,fail_accuracy,pass +selecsls42b,pass,pass,pass,pass,pass +spnasnet_100,pass,pass,pass,pass,pass swin_base_patch4_window7_224,pass,pass,pass,pass,pass -swsl_resnext101_32x16d,pass,fail_accuracy,pass,fail_accuracy,pass -tf_efficientnet_b0,pass,fail_accuracy,pass,fail_accuracy,pass -tf_mixnet_l,pass,fail_accuracy,pass,fail_accuracy,pass +swsl_resnext101_32x16d,pass,pass,pass,pass,pass +tf_efficientnet_b0,pass,pass,pass,pass,pass +tf_mixnet_l,pass,pass,pass,pass,pass tinynet_a,pass,pass,pass,pass,pass tnt_s_patch16_224,pass,pass,pass,pass,pass twins_pcpvt_base,pass,pass,pass,pass,pass -visformer_small,pass,fail_accuracy,pass,fail_accuracy,pass +visformer_small,pass,pass,pass,pass,pass vit_base_patch16_224,pass,pass,pass,pass,pass volo_d1_224,pass,pass,pass,pass,pass xcit_large_24_p8_224,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip diff --git a/.github/ci_expected_accuracy/inductor_torchbench_inference.csv b/.github/ci_expected_accuracy/inductor_torchbench_inference.csv index b182eba94..fe1914043 100644 --- a/.github/ci_expected_accuracy/inductor_torchbench_inference.csv +++ b/.github/ci_expected_accuracy/inductor_torchbench_inference.csv @@ -13,7 +13,7 @@ basic_gnn_sage,pass,pass,pass,pass,pass cm3leon_generate,pass,pass,pass,pass,pass dcgan,pass,pass,pass,pass,pass demucs,pass,pass,pass,pass,pass -densenet121,pass,pass,pass,fail_accuracy,pass +densenet121,pass,pass,pass,pass,pass detectron2_fasterrcnn_r_101_c4,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy detectron2_fasterrcnn_r_101_dc5,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy detectron2_fasterrcnn_r_101_fpn,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy @@ -32,7 +32,7 @@ doctr_reco_predictor,pass,pass,pass,pass,pass drq,pass,pass,pass,pass,pass fastNLP_Bert,pass,pass,pass,pass,pass functorch_dp_cifar10,pass,pass,pass,pass,pass -functorch_maml_omniglot,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +functorch_maml_omniglot,pass,pass,pass,pass,pass hf_Albert,pass,pass,pass,pass,pass hf_Bart,pass,pass,pass,pass,pass hf_Bert,pass,pass,pass,pass,pass @@ -55,17 +55,17 @@ llama,pass,pass,pass,pass,pass llama_v2_7b_16h,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip llava,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run maml,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip -maml_omniglot,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +maml_omniglot,pass,pass,pass,pass,pass microbench_unbacked_tolist_sum,pass,pass,pass,pass,pass -mnasnet1_0,pass,fail_accuracy,pass,fail_accuracy,pass -mobilenet_v2,pass,fail_accuracy,pass,fail_accuracy,pass +mnasnet1_0,pass,pass,pass,pass,pass +mobilenet_v2,pass,pass,pass,pass,pass mobilenet_v2_quantized_qat,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load mobilenet_v3_large,pass,pass,pass,pass,pass -moco,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load +moco,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run moondream,pass,pass,pass,pass,pass nanogpt,pass,pass,pass,pass,pass nvidia_deeprecommender,pass,pass,pass,pass,pass -opacus_cifar10,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +opacus_cifar10,pass,pass,pass,pass,pass phlippe_densenet,pass,pass,pass,pass,pass phlippe_resnet,pass,pass,pass,pass,pass pyhpc_equation_of_state,pass,pass,pass,pass,pass @@ -74,34 +74,33 @@ pyhpc_turbulent_kinetic_energy,pass,pass,pass,pass,pass pytorch_CycleGAN_and_pix2pix,pass,pass,pass,pass,pass pytorch_stargan,pass,pass,pass,pass,pass pytorch_unet,pass,pass,pass,pass,pass -resnet152,pass,fail_accuracy,pass,fail_accuracy,pass -resnet18,pass,pass,pass,fail_accuracy,pass +resnet152,pass,pass,pass,pass,pass +resnet18,pass,pass,pass,pass,pass resnet50,pass,pass,pass,pass,pass resnet50_quantized_qat,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load -resnext50_32x4d,pass,pass,pass,fail_accuracy,pass +resnext50_32x4d,pass,pass,pass,pass,pass sam,pass,pass,pass,pass,pass sam_fast,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run -shufflenet_v2_x1_0,pass,fail_accuracy,pass,fail_accuracy,pass +shufflenet_v2_x1_0,pass,pass,pass,pass,pass simple_gpt,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load simple_gpt_tp_manual,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load soft_actor_critic,pass,pass,pass,pass,pass speech_transformer,pass,pass,pass,pass,pass squeezenet1_1,pass,fail_accuracy,pass,pass,pass -stable_diffusion_text_encoder,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run -stable_diffusion_unet,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +stable_diffusion_text_encoder,pass,pass,pass,pass,pass +stable_diffusion_unet,eager_fail_to_run,pass_due_to_skip,pass_due_to_skip,eager_fail_to_run,eager_fail_to_run tacotron2,pass,pass,pass,fail_to_run,fail_to_run timm_efficientdet,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load timm_efficientnet,pass,pass,pass,pass,pass -timm_nfnet,pass,fail_accuracy,pass,pass,pass -timm_regnet,pass,pass,pass,fail_accuracy,pass -timm_resnest,pass,fail_accuracy,pass,fail_accuracy,pass +timm_nfnet,pass,pass,pass,pass,pass +timm_regnet,pass,pass,pass,pass,pass +timm_resnest,pass,pass,pass,pass,pass timm_vision_transformer,pass,pass,pass,pass,pass timm_vision_transformer_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip -timm_vovnet,pass,fail_accuracy,pass,fail_accuracy,pass +timm_vovnet,pass,pass,pass,pass,pass torch_multimodal_clip,pass,pass,pass,pass,pass tts_angular,pass,pass,pass,pass,pass vgg16,pass,pass,pass,pass,pass vision_maskrcnn,pass,pass,pass,eager_fail_to_run,eager_fail_to_run -# Skip yolov3 for known torchbench issue: https://github.com/intel/torch-xpu-ops/issues/1229 -yolov3,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +yolov3,pass,pass,pass,pass,pass hf_Roberta_base,pass,pass,pass,pass,pass diff --git a/.github/ci_expected_accuracy/inductor_torchbench_training.csv b/.github/ci_expected_accuracy/inductor_torchbench_training.csv index f4619f482..2ae6f9fd0 100644 --- a/.github/ci_expected_accuracy/inductor_torchbench_training.csv +++ b/.github/ci_expected_accuracy/inductor_torchbench_training.csv @@ -12,8 +12,8 @@ basic_gnn_gin,pass,pass,pass,pass,pass basic_gnn_sage,pass,pass,pass,pass,pass cm3leon_generate,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run dcgan,pass,pass,pass,pass,pass -demucs,fail_accuracy,eager_fail_to_run,eager_fail_to_run,fail_accuracy,fail_accuracy -densenet121,pass,pass,pass,fail_accuracy,pass +demucs,eager_two_runs_differ,eager_fail_to_run,eager_fail_to_run,eager_two_runs_differ,eager_two_runs_differ +densenet121,pass,pass,pass,pass,pass detectron2_fasterrcnn_r_101_c4,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run detectron2_fasterrcnn_r_101_dc5,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run detectron2_fasterrcnn_r_101_fpn,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run @@ -32,7 +32,7 @@ doctr_reco_predictor,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager drq,pass,pass,pass,pass,pass fastNLP_Bert,pass,pass,pass,pass,pass functorch_dp_cifar10,fail_accuracy,fail_accuracy,fail_accuracy,pass,pass -functorch_maml_omniglot,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +functorch_maml_omniglot,pass,pass,pass,fail_accuracy,pass hf_Albert,pass,pass,pass,pass,pass hf_Bart,pass,pass,pass,pass,pass hf_Bert,pass,pass,pass,pass,pass @@ -55,13 +55,13 @@ llama,pass,pass,pass,pass,pass llama_v2_7b_16h,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip llava,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run maml,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run -maml_omniglot,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +maml_omniglot,pass,pass,pass,pass,pass microbench_unbacked_tolist_sum,pass,pass,pass,pass,pass -mnasnet1_0,pass,fail_accuracy,pass,fail_accuracy,pass -mobilenet_v2,pass,fail_accuracy,pass,fail_accuracy,pass +mnasnet1_0,pass,pass,pass,pass,pass +mobilenet_v2,pass,pass,pass,pass,pass mobilenet_v2_quantized_qat,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run mobilenet_v3_large,pass,pass,pass,pass,pass -moco,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load +moco,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run moondream,pass,pass,pass,pass,pass nanogpt,pass,pass,pass,pass,pass nvidia_deeprecommender,pass,pass,pass,pass,pass @@ -74,34 +74,33 @@ pyhpc_turbulent_kinetic_energy,model_fail_to_load,model_fail_to_load,model_fail_ pytorch_CycleGAN_and_pix2pix,eager_two_runs_differ,eager_two_runs_differ,pass,eager_two_runs_differ,pass pytorch_stargan,pass,pass,pass,pass,pass pytorch_unet,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip -resnet152,pass,fail_accuracy,pass,fail_accuracy,pass -resnet18,pass,pass,pass,fail_accuracy,pass -resnet50,pass,pass,pass,fail_accuracy,pass +resnet152,pass,pass,pass,pass,pass +resnet18,pass,pass,pass,pass,pass +resnet50,pass,pass,pass,pass,pass resnet50_quantized_qat,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run -resnext50_32x4d,pass,pass,pass,fail_accuracy,pass +resnext50_32x4d,pass,pass,pass,pass,pass sam,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run sam_fast,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run -shufflenet_v2_x1_0,pass,fail_accuracy,pass,fail_accuracy,pass +shufflenet_v2_x1_0,pass,pass,pass,pass,pass simple_gpt,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load simple_gpt_tp_manual,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load soft_actor_critic,pass,pass,pass,pass,pass speech_transformer,pass,pass,pass,pass,pass squeezenet1_1,pass,pass,pass,pass,pass -stable_diffusion_text_encoder,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +stable_diffusion_text_encoder,pass,pass,pass,pass,pass stable_diffusion_unet,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run tacotron2,fail_to_run,fail_to_run,fail_to_run,fail_to_run,fail_to_run timm_efficientdet,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load timm_efficientnet,pass,pass,pass,pass,pass -timm_nfnet,pass,fail_accuracy,pass,pass,pass -timm_regnet,pass,fail_accuracy,fail_accuracy,fail_accuracy,fail_accuracy -timm_resnest,pass,fail_accuracy,pass,fail_accuracy,pass +timm_nfnet,pass,pass,pass,pass,pass +timm_regnet,pass,pass,pass,pass,pass +timm_resnest,pass,pass,pass,pass,pass timm_vision_transformer,pass,pass,pass,pass,pass timm_vision_transformer_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip -timm_vovnet,pass,fail_accuracy,pass,fail_accuracy,pass +timm_vovnet,pass,pass,pass,pass,pass torch_multimodal_clip,pass,pass,pass,pass,pass tts_angular,pass,pass,pass,pass,pass vgg16,pass,pass,pass,pass,pass vision_maskrcnn,pass,pass,pass,eager_fail_to_run,eager_fail_to_run -# Skip yolov3 for known torchbench issue: https://github.com/intel/torch-xpu-ops/issues/1229 yolov3,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run hf_Roberta_base,pass,pass,pass,pass,pass From 670807fa01a13bb53e469bf2c9932beaec0de829 Mon Sep 17 00:00:00 2001 From: mengfeil Date: Fri, 10 Jan 2025 16:39:37 +0800 Subject: [PATCH 4/6] Skip known issues --- .github/ci_expected_accuracy/check_expected.py | 3 ++- .../inductor_torchbench_inference.csv | 12 ++++++++---- .../inductor_torchbench_training.csv | 15 ++++++++++----- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/.github/ci_expected_accuracy/check_expected.py b/.github/ci_expected_accuracy/check_expected.py index 6d321e34b..7efde28d3 100644 --- a/.github/ci_expected_accuracy/check_expected.py +++ b/.github/ci_expected_accuracy/check_expected.py @@ -67,7 +67,8 @@ else: if refer_accuracy == "N/A": new_models.append([model_name, test_accuracy]) - real_failed_models.append([model_name, test_accuracy]) + # Not failed for new models + expected_failed_models.append([model_name, test_accuracy]) refer_data.loc[len(refer_data),:] = "N/A" refer_data.at[len(refer_data) - 1, "name"] = model_name refer_data.at[len(refer_data) - 1, args.dtype] = test_accuracy diff --git a/.github/ci_expected_accuracy/inductor_torchbench_inference.csv b/.github/ci_expected_accuracy/inductor_torchbench_inference.csv index fe1914043..a2b6d5b71 100644 --- a/.github/ci_expected_accuracy/inductor_torchbench_inference.csv +++ b/.github/ci_expected_accuracy/inductor_torchbench_inference.csv @@ -3,7 +3,8 @@ torchrec_dlrm,pass,eager_fail_to_run,eager_fail_to_run,fail_to_run,fail_to_run BERT_pytorch,pass,pass,pass,pass,pass Background_Matting,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip DALLE2_pytorch,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run -LearningToPaint,pass,pass,pass,pass,pass +# https://github.com/intel/torch-xpu-ops/issues/1263 +LearningToPaint,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run Super_SloMo,pass,pass,pass,pass,pass alexnet,pass,pass,pass,pass,pass basic_gnn_edgecnn,pass,pass,pass,pass,pass @@ -42,7 +43,8 @@ hf_DistilBert,pass,pass,pass,pass,pass hf_GPT2,pass,pass,pass,pass,pass hf_GPT2_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip hf_Longformer,pass,pass,pass,pass,pass -hf_Reformer,pass,pass,pass,pass,pass +# https://github.com/intel/torch-xpu-ops/issues/1262 +hf_Reformer,eager_two_runs_differ,eager_two_runs_differ,eager_two_runs_differ,eager_two_runs_differ,eager_two_runs_differ hf_T5,pass,pass,pass,pass,pass hf_T5_base,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run hf_T5_generate,pass,pass,pass,pass,pass @@ -64,7 +66,8 @@ mobilenet_v3_large,pass,pass,pass,pass,pass moco,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run moondream,pass,pass,pass,pass,pass nanogpt,pass,pass,pass,pass,pass -nvidia_deeprecommender,pass,pass,pass,pass,pass +# https://github.com/intel/torch-xpu-ops/issues/1260 +nvidia_deeprecommender,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run opacus_cifar10,pass,pass,pass,pass,pass phlippe_densenet,pass,pass,pass,pass,pass phlippe_resnet,pass,pass,pass,pass,pass @@ -84,7 +87,8 @@ sam_fast,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run shufflenet_v2_x1_0,pass,pass,pass,pass,pass simple_gpt,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load simple_gpt_tp_manual,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load -soft_actor_critic,pass,pass,pass,pass,pass +# https://github.com/intel/torch-xpu-ops/issues/1273 +soft_actor_critic,pass,fail_accuracy,pass,pass,pass speech_transformer,pass,pass,pass,pass,pass squeezenet1_1,pass,fail_accuracy,pass,pass,pass stable_diffusion_text_encoder,pass,pass,pass,pass,pass diff --git a/.github/ci_expected_accuracy/inductor_torchbench_training.csv b/.github/ci_expected_accuracy/inductor_torchbench_training.csv index 2ae6f9fd0..c091d24a0 100644 --- a/.github/ci_expected_accuracy/inductor_torchbench_training.csv +++ b/.github/ci_expected_accuracy/inductor_torchbench_training.csv @@ -3,8 +3,10 @@ torchrec_dlrm,pass,eager_fail_to_run,eager_fail_to_run,pass,pass BERT_pytorch,pass,pass,pass,pass,pass Background_Matting,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip DALLE2_pytorch,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run -LearningToPaint,pass,pass,pass,pass,pass -Super_SloMo,eager_two_runs_differ,pass,pass,pass,pass +# https://github.com/intel/torch-xpu-ops/issues/1263 +LearningToPaint,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +# https://github.com/intel/torch-xpu-ops/issues/1256 +Super_SloMo,eager_two_runs_differ,pass,pass,eager_two_runs_differ,pass alexnet,pass,pass,pass,pass,pass basic_gnn_edgecnn,pass,pass,pass,pass,pass basic_gnn_gcn,pass,pass,pass,pass,pass @@ -42,7 +44,8 @@ hf_DistilBert,pass,pass,pass,pass,pass hf_GPT2,pass,pass,pass,pass,pass hf_GPT2_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip hf_Longformer,pass,pass,pass,pass,pass -hf_Reformer,pass,pass,pass,pass,pass +# https://github.com/intel/torch-xpu-ops/issues/1262 +hf_Reformer,eager_two_runs_differ,eager_two_runs_differ,eager_two_runs_differ,eager_two_runs_differ,eager_two_runs_differ hf_T5,pass,pass,pass,pass,pass hf_T5_base,pass,pass,pass,pass,pass hf_T5_generate,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run @@ -64,7 +67,8 @@ mobilenet_v3_large,pass,pass,pass,pass,pass moco,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run moondream,pass,pass,pass,pass,pass nanogpt,pass,pass,pass,pass,pass -nvidia_deeprecommender,pass,pass,pass,pass,pass +# https://github.com/intel/torch-xpu-ops/issues/1260 +nvidia_deeprecommender,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run opacus_cifar10,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run phlippe_densenet,pass,pass,pass,pass,pass phlippe_resnet,pass,fail_accuracy,pass,fail_accuracy,pass @@ -101,6 +105,7 @@ timm_vovnet,pass,pass,pass,pass,pass torch_multimodal_clip,pass,pass,pass,pass,pass tts_angular,pass,pass,pass,pass,pass vgg16,pass,pass,pass,pass,pass -vision_maskrcnn,pass,pass,pass,eager_fail_to_run,eager_fail_to_run +# https://github.com/intel/torch-xpu-ops/issues/1264 +vision_maskrcnn,eager_fail_to_run,pass,pass,eager_fail_to_run,eager_fail_to_run yolov3,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run hf_Roberta_base,pass,pass,pass,pass,pass From b0b02f30b0b10ed8ca1401bccdb7b7e021ec0a21 Mon Sep 17 00:00:00 2001 From: mengfeil Date: Fri, 10 Jan 2025 17:12:08 +0800 Subject: [PATCH 5/6] Add known issue for fail_accuracy --- .../ci_expected_accuracy/inductor_timm_models_training.csv | 3 +++ .../ci_expected_accuracy/inductor_torchbench_inference.csv | 5 +++++ .../ci_expected_accuracy/inductor_torchbench_training.csv | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/.github/ci_expected_accuracy/inductor_timm_models_training.csv b/.github/ci_expected_accuracy/inductor_timm_models_training.csv index e723a49d3..86963c968 100644 --- a/.github/ci_expected_accuracy/inductor_timm_models_training.csv +++ b/.github/ci_expected_accuracy/inductor_timm_models_training.csv @@ -6,6 +6,7 @@ cait_m36_384,pass,pass,pass,pass,pass coat_lite_mini,pass,pass,pass,pass,pass convit_base,pass,pass,pass,pass,pass convmixer_768_32,pass,pass,pass,pass,pass +# https://github.com/intel/torch-xpu-ops/issues/1274 convnext_base,pass,fail_accuracy,fail_accuracy,pass,pass crossvit_9_240,pass,pass,pass,pass,pass cspdarknet53,pass,pass,pass,pass,pass @@ -14,9 +15,11 @@ dla102,pass,pass,pass,pass,pass dm_nfnet_f0,pass,pass,pass,pass,pass dpn107,pass,pass,pass,pass,pass eca_botnext26ts_256,pass,pass,pass,pass,pass +# https://github.com/intel/torch-xpu-ops/issues/1275 eca_halonext26ts,pass,pass,pass,fail_accuracy,pass ese_vovnet19b_dw,pass,pass,pass,pass,pass fbnetc_100,pass,pass,pass,pass,pass +# https://github.com/intel/torch-xpu-ops/issues/1275 fbnetv3_b,pass,pass,pass,fail_accuracy,pass gernet_l,pass,pass,pass,pass,pass ghostnet_100,pass,pass,pass,pass,pass diff --git a/.github/ci_expected_accuracy/inductor_torchbench_inference.csv b/.github/ci_expected_accuracy/inductor_torchbench_inference.csv index a2b6d5b71..5988681b5 100644 --- a/.github/ci_expected_accuracy/inductor_torchbench_inference.csv +++ b/.github/ci_expected_accuracy/inductor_torchbench_inference.csv @@ -1,4 +1,5 @@ name,float32,bfloat16,float16,amp_bf16,amp_fp16 +# https://github.com/intel/torch-xpu-ops/issues/1221 torchrec_dlrm,pass,eager_fail_to_run,eager_fail_to_run,fail_to_run,fail_to_run BERT_pytorch,pass,pass,pass,pass,pass Background_Matting,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip @@ -15,6 +16,7 @@ cm3leon_generate,pass,pass,pass,pass,pass dcgan,pass,pass,pass,pass,pass demucs,pass,pass,pass,pass,pass densenet121,pass,pass,pass,pass,pass +# https://github.com/intel/torch-xpu-ops/issues/1278 detectron2_fasterrcnn_r_101_c4,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy detectron2_fasterrcnn_r_101_dc5,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy detectron2_fasterrcnn_r_101_fpn,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy @@ -46,6 +48,7 @@ hf_Longformer,pass,pass,pass,pass,pass # https://github.com/intel/torch-xpu-ops/issues/1262 hf_Reformer,eager_two_runs_differ,eager_two_runs_differ,eager_two_runs_differ,eager_two_runs_differ,eager_two_runs_differ hf_T5,pass,pass,pass,pass,pass +# https://github.com/intel/torch-xpu-ops/issues/1276 hf_T5_base,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run hf_T5_generate,pass,pass,pass,pass,pass hf_T5_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip @@ -55,6 +58,7 @@ hf_distil_whisper,pass,pass,pass,pass,pass lennard_jones,pass,pass,pass,pass,pass llama,pass,pass,pass,pass,pass llama_v2_7b_16h,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip +# https://github.com/intel/torch-xpu-ops/issues/1277 llava,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run maml,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip maml_omniglot,pass,pass,pass,pass,pass @@ -92,6 +96,7 @@ soft_actor_critic,pass,fail_accuracy,pass,pass,pass speech_transformer,pass,pass,pass,pass,pass squeezenet1_1,pass,fail_accuracy,pass,pass,pass stable_diffusion_text_encoder,pass,pass,pass,pass,pass +# https://github.com/intel/torch-xpu-ops/issues/1261 stable_diffusion_unet,eager_fail_to_run,pass_due_to_skip,pass_due_to_skip,eager_fail_to_run,eager_fail_to_run tacotron2,pass,pass,pass,fail_to_run,fail_to_run timm_efficientdet,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load diff --git a/.github/ci_expected_accuracy/inductor_torchbench_training.csv b/.github/ci_expected_accuracy/inductor_torchbench_training.csv index c091d24a0..4e44a8f3c 100644 --- a/.github/ci_expected_accuracy/inductor_torchbench_training.csv +++ b/.github/ci_expected_accuracy/inductor_torchbench_training.csv @@ -33,7 +33,9 @@ doctr_det_predictor,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_ doctr_reco_predictor,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run drq,pass,pass,pass,pass,pass fastNLP_Bert,pass,pass,pass,pass,pass +# https://github.com/intel/torch-xpu-ops/issues/508 functorch_dp_cifar10,fail_accuracy,fail_accuracy,fail_accuracy,pass,pass +# https://github.com/intel/torch-xpu-ops/issues/510 functorch_maml_omniglot,pass,pass,pass,fail_accuracy,pass hf_Albert,pass,pass,pass,pass,pass hf_Bart,pass,pass,pass,pass,pass @@ -71,6 +73,7 @@ nanogpt,pass,pass,pass,pass,pass nvidia_deeprecommender,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run opacus_cifar10,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run phlippe_densenet,pass,pass,pass,pass,pass +# https://github.com/intel/torch-xpu-ops/issues/509 phlippe_resnet,pass,fail_accuracy,pass,fail_accuracy,pass pyhpc_equation_of_state,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run pyhpc_isoneutral_mixing,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run @@ -92,6 +95,7 @@ soft_actor_critic,pass,pass,pass,pass,pass speech_transformer,pass,pass,pass,pass,pass squeezenet1_1,pass,pass,pass,pass,pass stable_diffusion_text_encoder,pass,pass,pass,pass,pass +# https://github.com/intel/torch-xpu-ops/issues/1261 stable_diffusion_unet,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run tacotron2,fail_to_run,fail_to_run,fail_to_run,fail_to_run,fail_to_run timm_efficientdet,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load From dd6f80cde75c1d2de79e076265a872c3d708be43 Mon Sep 17 00:00:00 2001 From: mengfeil Date: Fri, 10 Jan 2025 17:18:26 +0800 Subject: [PATCH 6/6] comment --- .github/ci_expected_accuracy/check_expected.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/ci_expected_accuracy/check_expected.py b/.github/ci_expected_accuracy/check_expected.py index 7efde28d3..9c45e0a42 100644 --- a/.github/ci_expected_accuracy/check_expected.py +++ b/.github/ci_expected_accuracy/check_expected.py @@ -3,6 +3,7 @@ import pandas as pd import pathlib +# Reference last updated is https://github.com/intel/torch-xpu-ops/pull/1223 parser = argparse.ArgumentParser(description="Accuracy Check", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("--category", type=str, default="inductor", help="inductor")