Upload 17 files

results-len512/Classification/scores_amazon_counterfactual_classification.json

@@ -0,0 +1,23 @@
+{
+    "metric_name": "macro_f1",
+    "metric_value": 0.8212903958100242,
+    "details": {
+        "optimal_classifier_name": "logreg",
+        "val_scores": {
+            "knn_cosine_k_2": {
+                "accuracy": 0.9055793991416309,
+                "macro_f1": 0.6510076252723311
+            },
+            "logreg": {
+                "accuracy": 0.9184549356223176,
+                "macro_f1": 0.756611138600253
+            }
+        },
+        "test_scores": {
+            "logreg": {
+                "accuracy": 0.936830835117773,
+                "macro_f1": 0.8212903958100242
+            }
+        }
+    }
+}

results-len512/Classification/scores_amazon_review_classification.json

@@ -0,0 +1,23 @@
+{
+    "metric_name": "macro_f1",
+    "metric_value": 0.6134613719538808,
+    "details": {
+        "optimal_classifier_name": "logreg",
+        "val_scores": {
+            "knn_cosine_k_2": {
+                "accuracy": 0.456,
+                "macro_f1": 0.44947743361535253
+            },
+            "logreg": {
+                "accuracy": 0.6106,
+                "macro_f1": 0.6067084288597717
+            }
+        },
+        "test_scores": {
+            "logreg": {
+                "accuracy": 0.6164,
+                "macro_f1": 0.6134613719538808
+            }
+        }
+    }
+}

results-len512/Classification/scores_massive_intent_classification.json

@@ -0,0 +1,23 @@
+{
+    "metric_name": "macro_f1",
+    "metric_value": 0.821659588084153,
+    "details": {
+        "optimal_classifier_name": "logreg",
+        "val_scores": {
+            "knn_cosine_k_2": {
+                "accuracy": 0.7830791933103788,
+                "macro_f1": 0.7540349043356395
+            },
+            "logreg": {
+                "accuracy": 0.8558780127889818,
+                "macro_f1": 0.8412198611245395
+            }
+        },
+        "test_scores": {
+            "logreg": {
+                "accuracy": 0.8493611297915266,
+                "macro_f1": 0.821659588084153
+            }
+        }
+    }
+}

results-len512/Classification/scores_massive_scenario_classification.json

@@ -0,0 +1,23 @@
+{
+    "metric_name": "macro_f1",
+    "metric_value": 0.8927605477819499,
+    "details": {
+        "optimal_classifier_name": "logreg",
+        "val_scores": {
+            "knn_cosine_k_2": {
+                "accuracy": 0.8612887358583374,
+                "macro_f1": 0.8475812013823129
+            },
+            "logreg": {
+                "accuracy": 0.8957206099360551,
+                "macro_f1": 0.889349188280443
+            }
+        },
+        "test_scores": {
+            "logreg": {
+                "accuracy": 0.8927370544720915,
+                "macro_f1": 0.8927605477819499
+            }
+        }
+    }
+}

results-len512/Clustering/scores_livedoor_news.json

@@ -0,0 +1,36 @@
+{
+    "metric_name": "v_measure_score",
+    "metric_value": 0.5937524124850755,
+    "details": {
+        "optimal_clustering_model_name": "MiniBatchKMeans",
+        "val_scores": {
+            "MiniBatchKMeans": {
+                "v_measure_score": 0.5932473997172523,
+                "homogeneity_score": 0.5844833525834573,
+                "completeness_score": 0.6022782731375254
+            },
+            "AgglomerativeClustering": {
+                "v_measure_score": 0.5823205096909835,
+                "homogeneity_score": 0.5730383501428752,
+                "completeness_score": 0.5919083279167753
+            },
+            "BisectingKMeans": {
+                "v_measure_score": 0.5619252755091108,
+                "homogeneity_score": 0.5612257535302629,
+                "completeness_score": 0.5626265434579824
+            },
+            "Birch": {
+                "v_measure_score": 0.5824759855704932,
+                "homogeneity_score": 0.5734728844465294,
+                "completeness_score": 0.5917662794981928
+            }
+        },
+        "test_scores": {
+            "MiniBatchKMeans": {
+                "v_measure_score": 0.5937524124850755,
+                "homogeneity_score": 0.5867837845991711,
+                "completeness_score": 0.6008885481501399
+            }
+        }
+    }
+}

results-len512/Clustering/scores_mewsc16.json

@@ -0,0 +1,36 @@
+{
+    "metric_name": "v_measure_score",
+    "metric_value": 0.4938330752654621,
+    "details": {
+        "optimal_clustering_model_name": "Birch",
+        "val_scores": {
+            "MiniBatchKMeans": {
+                "v_measure_score": 0.5020526188374297,
+                "homogeneity_score": 0.5508022841005213,
+                "completeness_score": 0.46123063979330864
+            },
+            "AgglomerativeClustering": {
+                "v_measure_score": 0.5567865030669769,
+                "homogeneity_score": 0.5991745068307929,
+                "completeness_score": 0.5199996459830565
+            },
+            "BisectingKMeans": {
+                "v_measure_score": 0.47357487467256326,
+                "homogeneity_score": 0.5170167479146354,
+                "completeness_score": 0.4368674730918793
+            },
+            "Birch": {
+                "v_measure_score": 0.5639601753239182,
+                "homogeneity_score": 0.6063086319538272,
+                "completeness_score": 0.527141271397583
+            }
+        },
+        "test_scores": {
+            "Birch": {
+                "v_measure_score": 0.4938330752654621,
+                "homogeneity_score": 0.5270581166850505,
+                "completeness_score": 0.4645485534255365
+            }
+        }
+    }
+}

results-len512/PairClassification/scores_paws_x_ja.json

@@ -0,0 +1,41 @@
+{
+    "metric_name": "binary_f1",
+    "metric_value": 0.6259758694109298,
+    "details": {
+        "optimal_distance_metric": "dot_similarities",
+        "val_scores": {
+            "cosine_distances": {
+                "accuracy": 0.5725,
+                "accuracy_threshold": 0.7325241565704346,
+                "binary_f1": 0.5979670522257273,
+                "binary_f1_threshold": 1.0
+            },
+            "manhatten_distances": {
+                "accuracy": 0.6105,
+                "accuracy_threshold": 63.98142623901367,
+                "binary_f1": 0.6014825273561596,
+                "binary_f1_threshold": 395.8541259765625
+            },
+            "euclidean_distances": {
+                "accuracy": 0.6115,
+                "accuracy_threshold": 2.9641363620758057,
+                "binary_f1": 0.6016949152542372,
+                "binary_f1_threshold": 18.21161460876465
+            },
+            "dot_similarities": {
+                "accuracy": 0.583,
+                "accuracy_threshold": 872.2387084960938,
+                "binary_f1": 0.6023329798515377,
+                "binary_f1_threshold": 735.9739990234375
+            }
+        },
+        "test_scores": {
+            "dot_similarities": {
+                "accuracy": 0.581,
+                "accuracy_threshold": 872.2387084960938,
+                "binary_f1": 0.6259758694109298,
+                "binary_f1_threshold": 735.9739990234375
+            }
+        }
+    }
+}

results-len512/Reranking/scores_esci.json

@@ -0,0 +1,31 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.934131495980383,
+    "details": {
+        "optimal_distance_metric": "cosine_similarity",
+        "val_scores": {
+            "cosine_similarity": {
+                "ndcg@10": 0.947221947678768,
+                "ndcg@20": 0.9580109255686136,
+                "ndcg@40": 0.9655483305547552
+            },
+            "dot_score": {
+                "ndcg@10": 0.939423095128925,
+                "ndcg@20": 0.9518546213913051,
+                "ndcg@40": 0.9602741048502043
+            },
+            "euclidean_distance": {
+                "ndcg@10": 0.946722927636254,
+                "ndcg@20": 0.9575361305939064,
+                "ndcg@40": 0.9650712902932668
+            }
+        },
+        "test_scores": {
+            "cosine_similarity": {
+                "ndcg@10": 0.934131495980383,
+                "ndcg@20": 0.9502309541380383,
+                "ndcg@40": 0.9589594356406169
+            }
+        }
+    }
+}

results-len512/Retrieval/scores_jagovfaqs_22k.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.7790411361538316,
+    "details": {
+        "optimal_distance_metric": "cosine_similarity",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.6557472945305645,
+                "accuracy@3": 0.8104708979233695,
+                "accuracy@5": 0.8531734425270547,
+                "accuracy@10": 0.896168470312957,
+                "ndcg@10": 0.7795641799225166,
+                "mrr@10": 0.7418103548331685
+            },
+            "dot_score": {
+                "accuracy@1": 0.49166422930681486,
+                "accuracy@3": 0.6809008482012284,
+                "accuracy@5": 0.741152383737935,
+                "accuracy@10": 0.804913717461246,
+                "ndcg@10": 0.6488989893840936,
+                "mrr@10": 0.5988684847049854
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.6513600467973092,
+                "accuracy@3": 0.8110558642878034,
+                "accuracy@5": 0.8517110266159695,
+                "accuracy@10": 0.8964609534951741,
+                "ndcg@10": 0.7780790057668981,
+                "mrr@10": 0.7397305904910466
+            }
+        },
+        "test_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.6494152046783626,
+                "accuracy@3": 0.8093567251461988,
+                "accuracy@5": 0.8555555555555555,
+                "accuracy@10": 0.9008771929824562,
+                "ndcg@10": 0.7790411361538316,
+                "mrr@10": 0.7395963287849251
+            }
+        }
+    }
+}

results-len512/Retrieval/scores_jaqket.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.7001158161632123,
+    "details": {
+        "optimal_distance_metric": "cosine_similarity",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.521608040201005,
+                "accuracy@3": 0.7266331658291457,
+                "accuracy@5": 0.7839195979899497,
+                "accuracy@10": 0.8361809045226131,
+                "ndcg@10": 0.6835031423555805,
+                "mrr@10": 0.634055196618011
+            },
+            "dot_score": {
+                "accuracy@1": 0.46934673366834173,
+                "accuracy@3": 0.6532663316582915,
+                "accuracy@5": 0.714572864321608,
+                "accuracy@10": 0.7788944723618091,
+                "ndcg@10": 0.6249364714039314,
+                "mrr@10": 0.5755304299274148
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.5135678391959799,
+                "accuracy@3": 0.7206030150753768,
+                "accuracy@5": 0.7758793969849246,
+                "accuracy@10": 0.8311557788944723,
+                "ndcg@10": 0.6775263671411185,
+                "mrr@10": 0.6277366993698654
+            }
+        },
+        "test_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.526579739217653,
+                "accuracy@3": 0.7512537612838516,
+                "accuracy@5": 0.8164493480441324,
+                "accuracy@10": 0.8585757271815446,
+                "ndcg@10": 0.7001158161632123,
+                "mrr@10": 0.6483888172453874
+            }
+        }
+    }
+}

results-len512/Retrieval/scores_mrtydi.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.48767396659564394,
+    "details": {
+        "optimal_distance_metric": "euclidean_distance",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.38038793103448276,
+                "accuracy@3": 0.5668103448275862,
+                "accuracy@5": 0.6357758620689655,
+                "accuracy@10": 0.7079741379310345,
+                "ndcg@10": 0.5410435939581002,
+                "mrr@10": 0.48780617131910187
+            },
+            "dot_score": {
+                "accuracy@1": 0.21443965517241378,
+                "accuracy@3": 0.36961206896551724,
+                "accuracy@5": 0.4375,
+                "accuracy@10": 0.5334051724137931,
+                "ndcg@10": 0.3646669540665438,
+                "mrr@10": 0.31187183565955123
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.3857758620689655,
+                "accuracy@3": 0.5657327586206896,
+                "accuracy@5": 0.6379310344827587,
+                "accuracy@10": 0.709051724137931,
+                "ndcg@10": 0.5445163825100324,
+                "mrr@10": 0.49207674808429097
+            }
+        },
+        "test_scores": {
+            "euclidean_distance": {
+                "accuracy@1": 0.3680555555555556,
+                "accuracy@3": 0.5333333333333333,
+                "accuracy@5": 0.6069444444444444,
+                "accuracy@10": 0.6888888888888889,
+                "ndcg@10": 0.48767396659564394,
+                "mrr@10": 0.46919367283950547
+            }
+        }
+    }
+}

results-len512/Retrieval/scores_nlp_journal_abs_intro.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.9550969239533367,
+    "details": {
+        "optimal_distance_metric": "euclidean_distance",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.9098360655737705,
+                "accuracy@3": 0.9672131147540983,
+                "accuracy@5": 0.9918032786885246,
+                "accuracy@10": 0.9918032786885246,
+                "ndcg@10": 0.9519753703614401,
+                "mrr@10": 0.9387978142076502
+            },
+            "dot_score": {
+                "accuracy@1": 0.8278688524590164,
+                "accuracy@3": 0.9344262295081968,
+                "accuracy@5": 0.9672131147540983,
+                "accuracy@10": 0.9836065573770492,
+                "ndcg@10": 0.9117419934645401,
+                "mrr@10": 0.8880692167577415
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.9098360655737705,
+                "accuracy@3": 0.9672131147540983,
+                "accuracy@5": 0.9918032786885246,
+                "accuracy@10": 0.9918032786885246,
+                "ndcg@10": 0.955194954465656,
+                "mrr@10": 0.942896174863388
+            }
+        },
+        "test_scores": {
+            "euclidean_distance": {
+                "accuracy@1": 0.9105691056910569,
+                "accuracy@3": 0.9735772357723578,
+                "accuracy@5": 0.9817073170731707,
+                "accuracy@10": 0.991869918699187,
+                "ndcg@10": 0.9550969239533367,
+                "mrr@10": 0.9428547231900889
+            }
+        }
+    }
+}

results-len512/Retrieval/scores_nlp_journal_title_abs.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.9800724651571329,
+    "details": {
+        "optimal_distance_metric": "cosine_similarity",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.9590163934426229,
+                "accuracy@3": 0.9754098360655737,
+                "accuracy@5": 0.9836065573770492,
+                "accuracy@10": 1.0,
+                "ndcg@10": 0.9787291019625602,
+                "mrr@10": 0.9719945355191256
+            },
+            "dot_score": {
+                "accuracy@1": 0.8770491803278688,
+                "accuracy@3": 0.9754098360655737,
+                "accuracy@5": 0.9918032786885246,
+                "accuracy@10": 1.0,
+                "ndcg@10": 0.9465822466240205,
+                "mrr@10": 0.9285519125683058
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.9590163934426229,
+                "accuracy@3": 0.9754098360655737,
+                "accuracy@5": 0.9836065573770492,
+                "accuracy@10": 1.0,
+                "ndcg@10": 0.9782768299015435,
+                "mrr@10": 0.9715391621129326
+            }
+        },
+        "test_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.959349593495935,
+                "accuracy@3": 0.9898373983739838,
+                "accuracy@5": 0.991869918699187,
+                "accuracy@10": 0.9939024390243902,
+                "ndcg@10": 0.9800724651571329,
+                "mrr@10": 0.9753274616079494
+            }
+        }
+    }
+}

results-len512/Retrieval/scores_nlp_journal_title_intro.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.8726100026859497,
+    "details": {
+        "optimal_distance_metric": "euclidean_distance",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.7868852459016393,
+                "accuracy@3": 0.8770491803278688,
+                "accuracy@5": 0.9180327868852459,
+                "accuracy@10": 0.9590163934426229,
+                "ndcg@10": 0.8714655319010017,
+                "mrr@10": 0.8436020036429872
+            },
+            "dot_score": {
+                "accuracy@1": 0.6967213114754098,
+                "accuracy@3": 0.860655737704918,
+                "accuracy@5": 0.8852459016393442,
+                "accuracy@10": 0.9180327868852459,
+                "ndcg@10": 0.8111445047740787,
+                "mrr@10": 0.7763173302107729
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.7950819672131147,
+                "accuracy@3": 0.9016393442622951,
+                "accuracy@5": 0.9180327868852459,
+                "accuracy@10": 0.9590163934426229,
+                "ndcg@10": 0.8753147176163506,
+                "mrr@10": 0.8488030184751496
+            }
+        },
+        "test_scores": {
+            "euclidean_distance": {
+                "accuracy@1": 0.774390243902439,
+                "accuracy@3": 0.9085365853658537,
+                "accuracy@5": 0.943089430894309,
+                "accuracy@10": 0.9613821138211383,
+                "ndcg@10": 0.8726100026859497,
+                "mrr@10": 0.8434160859465736
+            }
+        }
+    }
+}

results-len512/STS/scores_jsick.json

@@ -0,0 +1,31 @@
+{
+    "metric_name": "spearman",
+    "metric_value": 0.7812165639898206,
+    "details": {
+        "optimal_similarity_metric": "manhatten_distance",
+        "val_scores": {
+            "cosine_similarity": {
+                "pearson": 0.8081424308280184,
+                "spearman": 0.7858960474676613
+            },
+            "manhatten_distance": {
+                "pearson": 0.8149453883286166,
+                "spearman": 0.7866688378992329
+            },
+            "euclidean_distance": {
+                "pearson": 0.8149453883286166,
+                "spearman": 0.7866688378992329
+            },
+            "dot_score": {
+                "pearson": 0.7289190000591521,
+                "spearman": 0.6947631991077574
+            }
+        },
+        "test_scores": {
+            "manhatten_distance": {
+                "pearson": 0.8101410164828766,
+                "spearman": 0.7812165639898206
+            }
+        }
+    }
+}

results-len512/STS/scores_jsts.json

@@ -0,0 +1,31 @@
+{
+    "metric_name": "spearman",
+    "metric_value": 0.8432397778118456,
+    "details": {
+        "optimal_similarity_metric": "manhatten_distance",
+        "val_scores": {
+            "cosine_similarity": {
+                "pearson": 0.8535555729076842,
+                "spearman": 0.819115099484669
+            },
+            "manhatten_distance": {
+                "pearson": 0.8633943133209987,
+                "spearman": 0.8261879574678256
+            },
+            "euclidean_distance": {
+                "pearson": 0.8633943133209987,
+                "spearman": 0.8261879574678256
+            },
+            "dot_score": {
+                "pearson": 0.713304993887736,
+                "spearman": 0.6363102927036544
+            }
+        },
+        "test_scores": {
+            "manhatten_distance": {
+                "pearson": 0.8819766195104106,
+                "spearman": 0.8432397778118456
+            }
+        }
+    }
+}

results-len512/summary.json

@@ -0,0 +1,62 @@
+{
+    "Classification": {
+        "amazon_counterfactual_classification": {
+            "macro_f1": 0.8212903958100242
+        },
+        "amazon_review_classification": {
+            "macro_f1": 0.6134613719538808
+        },
+        "massive_intent_classification": {
+            "macro_f1": 0.821659588084153
+        },
+        "massive_scenario_classification": {
+            "macro_f1": 0.8927605477819499
+        }
+    },
+    "Reranking": {
+        "esci": {
+            "ndcg@10": 0.934131495980383
+        }
+    },
+    "Retrieval": {
+        "jagovfaqs_22k": {
+            "ndcg@10": 0.7790411361538316
+        },
+        "jaqket": {
+            "ndcg@10": 0.7001158161632123
+        },
+        "mrtydi": {
+            "ndcg@10": 0.48767396659564394
+        },
+        "nlp_journal_abs_intro": {
+            "ndcg@10": 0.9550969239533367
+        },
+        "nlp_journal_title_abs": {
+            "ndcg@10": 0.9800724651571329
+        },
+        "nlp_journal_title_intro": {
+            "ndcg@10": 0.8726100026859497
+        }
+    },
+    "STS": {
+        "jsick": {
+            "spearman": 0.7812165639898206
+        },
+        "jsts": {
+            "spearman": 0.8432397778118456
+        }
+    },
+    "Clustering": {
+        "livedoor_news": {
+            "v_measure_score": 0.5937524124850755
+        },
+        "mewsc16": {
+            "v_measure_score": 0.4938330752654621
+        }
+    },
+    "PairClassification": {
+        "paws_x_ja": {
+            "binary_f1": 0.6259758694109298
+        }
+    }
+}