From 942afcab0d8e668f44fa3f9e496e82601e7e8f8a Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Mon, 2 Jun 2025 08:57:04 +0200 Subject: [PATCH 01/16] update did script and config --- .../src/montecover/did/did_pa_multi.py | 44 ++++--------------- scripts/did/did_pa_multi_config.yml | 28 ++++++++++-- 2 files changed, 33 insertions(+), 39 deletions(-) diff --git a/monte-cover/src/montecover/did/did_pa_multi.py b/monte-cover/src/montecover/did/did_pa_multi.py index fa6aeb6..eb84934 100644 --- a/monte-cover/src/montecover/did/did_pa_multi.py +++ b/monte-cover/src/montecover/did/did_pa_multi.py @@ -4,10 +4,9 @@ import numpy as np import pandas as pd from doubleml.did.datasets import make_did_CS2021 -from lightgbm import LGBMClassifier, LGBMRegressor -from sklearn.linear_model import LinearRegression, LogisticRegression from montecover.base import BaseSimulation +from montecover.utils import create_learner_from_config class DIDMultiCoverageSimulation(BaseSimulation): @@ -36,39 +35,13 @@ def __init__( def _process_config_parameters(self): """Process simulation-specific parameters from config""" # Process ML models in parameter grid + # Process ML models in parameter grid + assert "learners" in self.dml_parameters, "No learners specified in the config file" - assert ( - "learners" in self.dml_parameters - ), "No learners specified in the config file" + required_learners = ["ml_g", "ml_m"] for learner in self.dml_parameters["learners"]: - assert "ml_g" in learner, "No ml_g specified in the config file" - assert "ml_m" in learner, "No ml_m specified in the config file" - - # Convert ml_g strings to actual objects - if learner["ml_g"][0] == "Linear": - learner["ml_g"] = ("Linear", LinearRegression()) - elif learner["ml_g"][0] == "LGBM": - learner["ml_g"] = ( - "LGBM", - LGBMRegressor( - n_estimators=500, learning_rate=0.02, verbose=-1, n_jobs=1 - ), - ) - else: - raise ValueError(f"Unknown learner type: {learner['ml_g']}") - - # Convert ml_m strings to actual objects - if learner["ml_m"][0] == "Linear": - learner["ml_m"] = ("Linear", LogisticRegression()) - elif learner["ml_m"][0] == "LGBM": - learner["ml_m"] = ( - "LGBM", - LGBMClassifier( - n_estimators=500, learning_rate=0.02, verbose=-1, n_jobs=1 - ), - ) - else: - raise ValueError(f"Unknown learner type: {learner['ml_m']}") + for ml in required_learners: + assert ml in learner, f"No {ml} specified in the config file" def _calculate_oracle_values(self): """Calculate oracle values for the simulation.""" @@ -102,8 +75,9 @@ def _calculate_oracle_values(self): def run_single_rep(self, dml_data, dml_params) -> Dict[str, Any]: """Run a single repetition with the given parameters.""" # Extract parameters - learner_g_name, ml_g = dml_params["learners"]["ml_g"] - learner_m_name, ml_m = dml_params["learners"]["ml_m"] + learner_config = dml_params["learners"] + learner_g_name, ml_g = create_learner_from_config(learner_config["ml_g"]) + learner_m_name, ml_m = create_learner_from_config(learner_config["ml_m"]) score = dml_params["score"] in_sample_normalization = dml_params["in_sample_normalization"] diff --git a/scripts/did/did_pa_multi_config.yml b/scripts/did/did_pa_multi_config.yml index 67eead1..c89ef8d 100644 --- a/scripts/did/did_pa_multi_config.yml +++ b/scripts/did/did_pa_multi_config.yml @@ -10,13 +10,33 @@ dgp_parameters: DGP: [1, 4, 6] # Different DGP specifications n_obs: [2000] # Sample size for each simulation (has to be a list) +# Define reusable learner configurations +learner_definitions: + linear: &linear + name: "Linear" + + logistic: &logistic + name: "Logistic" + + lgbmr: &lgbmr + name: "LGBM Regr." + params: + n_estimators: 500 + learning_rate: 0.02 + + lgbmc: &lgbmc + name: "LGBM Clas." + params: + n_estimators: 500 + learning_rate: 0.02 + dml_parameters: # ML methods for ml_g and ml_m learners: - - ml_g: ["Linear"] - ml_m: ["Linear"] - - ml_g: ["LGBM"] - ml_m: ["LGBM"] + - ml_g: *linear + ml_m: *logistic + - ml_g: *lgbmr + ml_m: *lgbmc score: - observational # Standard DML score From 83c562f56e5b0f836f3aff0739cf6404afed95a0 Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Mon, 2 Jun 2025 10:55:35 +0200 Subject: [PATCH 02/16] add rdd coverage class to monte-cover --- monte-cover/src/montecover/rdd/__init__.py | 7 + monte-cover/src/montecover/rdd/rdd.py | 235 +++++++++++++++++++++ monte-cover/src/montecover/utils.py | 28 ++- 3 files changed, 268 insertions(+), 2 deletions(-) create mode 100644 monte-cover/src/montecover/rdd/__init__.py create mode 100644 monte-cover/src/montecover/rdd/rdd.py diff --git a/monte-cover/src/montecover/rdd/__init__.py b/monte-cover/src/montecover/rdd/__init__.py new file mode 100644 index 0000000..50efadb --- /dev/null +++ b/monte-cover/src/montecover/rdd/__init__.py @@ -0,0 +1,7 @@ +"""Monte Carlo coverage simulations for RDD.""" + +from montecover.rdd.rdd import RDDCoverageSimulation + +__all__ = [ + "RDDCoverageSimulation", +] diff --git a/monte-cover/src/montecover/rdd/rdd.py b/monte-cover/src/montecover/rdd/rdd.py new file mode 100644 index 0000000..5faaab7 --- /dev/null +++ b/monte-cover/src/montecover/rdd/rdd.py @@ -0,0 +1,235 @@ +import time +import warnings +from itertools import product +from typing import Any, Dict, Optional + +import doubleml as dml +import numpy as np +import pandas as pd +from doubleml.rdd.datasets import make_simple_rdd_data +from rdrobust import rdrobust +from statsmodels.nonparametric.kernel_regression import KernelReg + +from montecover.base import BaseSimulation +from montecover.utils import create_learner_from_config + + +class RDDCoverageSimulation(BaseSimulation): + """Simulation class for coverage properties of DoubleML RDFlex for RDD.""" + + def __init__( + self, + config_file: str, + suppress_warnings: bool = True, + log_level: str = "INFO", + log_file: Optional[str] = None, + ): + super().__init__( + config_file=config_file, + suppress_warnings=suppress_warnings, + log_level=log_level, + log_file=log_file, + ) + + self.fuzzy = self.dgp_parameters.get("fuzzy", [False])[0] + self.cutoff = self.dgp_parameters.get("cutoff", [0.0])[0] + # Calculate oracle values + self._calculate_oracle_values() + + def _process_config_parameters(self): + """Process simulation-specific parameters from config.""" + + # Process ML models in parameter grid + assert "learners" in self.dml_parameters, "No learners specified in the config file" + + required_learners = ["ml_g"] + for learner in self.dml_parameters["learners"]: + for ml in required_learners: + assert ml in learner, f"No {ml} specified in the config file" + + def _calculate_oracle_values(self): + """Calculate oracle values for the simulation.""" + self.logger.info("Calculating oracle values") + + data_oracle = make_simple_rdd_data(n_obs=int(1e5), fuzzy=self.fuzzy, cutoff=self.cutoff) + # get oracle value + score = data_oracle["score"] + ite = data_oracle["oracle_values"]["Y1"] - data_oracle["oracle_values"]["Y0"] + + # subset score and ite for faster computation + score_subset = (score >= (self.cutoff - 0.02)) & (score <= (self.cutoff + 0.02)) + self.logger.info(f"Oracle score subset size: {np.sum(score_subset)}") + kernel_reg = KernelReg(endog=ite[score_subset], exog=score[score_subset], var_type="c", reg_type="ll") + effect_at_cutoff, _ = kernel_reg.fit(np.array([self.cutoff])) + oracle_effect = effect_at_cutoff[0] + + self.logger.info(f"Oracle effect at cutoff: {oracle_effect}") + self.oracle_values = dict() + self.oracle_values["theta"] = oracle_effect + + def _process_repetition(self, i_rep): + """Process a single repetition with all parameter combinations.""" + if self.suppress_warnings: + warnings.simplefilter(action="ignore", category=UserWarning) + + i_param_comb = 0 + rep_results = { + "coverage": [], + } + + # loop through all parameter combinations + for dgp_param_values in product(*self.dgp_parameters.values()): + dgp_params = dict(zip(self.dgp_parameters.keys(), dgp_param_values)) + dml_data = self._generate_dml_data(dgp_params) + + # --- Run rdrobust benchmark --- + self.logger.debug(f"Rep {i_rep+1}: Running rdrobust benchmark for DGP {dgp_params}") + param_start_time_rd_benchmark = time.time() + + # Call the dedicated benchmark function + # Pass dml_data, current dgp_params, and repetition index + benchmark_result_list = self._rdrobust_benchmark(dml_data, dgp_params, i_rep) + if benchmark_result_list: + rep_results["coverage"].extend(benchmark_result_list) + + param_duration_rd_benchmark = time.time() - param_start_time_rd_benchmark + self.logger.debug(f"rdrobust benchmark for DGP {dgp_params} completed in {param_duration_rd_benchmark:.2f}s") + + for dml_param_values in product(*self.dml_parameters.values()): + dml_params = dict(zip(self.dml_parameters.keys(), dml_param_values)) + i_param_comb += 1 + + comb_results = self._process_parameter_combination(i_rep, i_param_comb, dgp_params, dml_params, dml_data) + rep_results["coverage"].extend(comb_results["coverage"]) + + return rep_results + + def _rdrobust_benchmark(self, dml_data, dml_params, i_rep): + """Run a benchmark using rdrobust for RDD.""" + + # Extract parameters + score = dml_data.data[dml_data.s_col] + Y = dml_data.data[dml_data.y_col] + Z = dml_data.data[dml_data.x_cols] + + benchmark_results_list = [] + for level in self.confidence_parameters["level"]: + rd_model = rdrobust(y=Y, x=score, covs=Z, c=self.cutoff, level=level * 100) + coef_rd = rd_model.coef.loc["Robust", "Coeff"] + ci_lower_rd = rd_model.ci.loc["Robust", "CI Lower"] + ci_upper_rd = rd_model.ci.loc["Robust", "CI Upper"] + + confint_for_compute = pd.DataFrame({"lower": [ci_lower_rd], "upper": [ci_upper_rd]}) + theta_for_compute = np.array([coef_rd]) + + coverage_metrics = self._compute_coverage( + thetas=theta_for_compute, + oracle_thetas=self.oracle_values["theta"], + confint=confint_for_compute, + joint_confint=None, + ) + + # Add metadata + coverage_metrics.update( + { + "repetition": i_rep, + "Learner g": "Linear", + "Learner m": "Logistic", + "Method": "rdrobust", + "fs_specification": "cutoff", + "level": level, + } + ) + benchmark_results_list.append(coverage_metrics) + + return benchmark_results_list + + def run_single_rep(self, dml_data, dml_params) -> Dict[str, Any]: + """Run a single repetition with the given parameters.""" + + # Extract parameters + learner_config = dml_params["learners"] + learner_g_name, ml_g = create_learner_from_config(learner_config["ml_g"]) + if self.fuzzy: + learner_m_name, ml_m = create_learner_from_config(learner_config["ml_m"]) + else: + learner_m_name, ml_m = "N/A", None + fs_specification = dml_params["fs_specification"] + + # Model + dml_model = dml.rdd.RDFlex( + obj_dml_data=dml_data, + ml_g=ml_g, + ml_m=ml_m, + n_folds=5, + n_rep=1, + fuzzy=self.fuzzy, + cutoff=self.cutoff, + fs_specification=fs_specification, + ) + dml_model.fit() + + result = { + "coverage": [], + } + for level in self.confidence_parameters["level"]: + level_result = dict() + level_result["coverage"] = self._compute_coverage( + thetas=dml_model.coef, + oracle_thetas=self.oracle_values["theta"], + confint=dml_model.confint(level=level), + joint_confint=None, + ) + + # add parameters to the result + for res in level_result.values(): + res.update( + { + "Learner g": learner_g_name, + "Learner m": learner_m_name, + "Method": "RDFlex", + "fs_specification": fs_specification, + "level": level, + } + ) + for key, res in level_result.items(): + result[key].append(res) + + return result + + def summarize_results(self): + """Summarize the simulation results.""" + self.logger.info("Summarizing simulation results") + + # Group by parameter combinations + groupby_cols = ["Method", "fs_specification", "Learner g", "Learner m", "level"] + aggregation_dict = { + "Coverage": "mean", + "CI Length": "mean", + "Bias": "mean", + "repetition": "count", + } + + # Aggregate results (possibly multiple result dfs) + result_summary = dict() + for result_name, result_df in self.results.items(): + result_summary[result_name] = result_df.groupby(groupby_cols).agg(aggregation_dict).reset_index() + self.logger.debug(f"Summarized {result_name} results") + + return result_summary + + def _generate_dml_data(self, dgp_params) -> dml.DoubleMLData: + """Generate data for the simulation.""" + data = make_simple_rdd_data( + n_obs=dgp_params["n_obs"], + fuzzy=dgp_params["fuzzy"], + cutoff=dgp_params["cutoff"], + ) + + score = data["score"] + Y = data["Y"] + X = data["X"].reshape(dgp_params["n_obs"], -1) + D = data["D"] + + dml_data = dml.DoubleMLData.from_arrays(y=Y, d=D, x=X, s=score) + return dml_data diff --git a/monte-cover/src/montecover/utils.py b/monte-cover/src/montecover/utils.py index b73dee4..dfadbbf 100644 --- a/monte-cover/src/montecover/utils.py +++ b/monte-cover/src/montecover/utils.py @@ -1,8 +1,9 @@ from typing import Any, Callable, Dict, Tuple +from doubleml.utils import GlobalRegressor from lightgbm import LGBMClassifier, LGBMRegressor -from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor -from sklearn.linear_model import LassoCV, LinearRegression, LogisticRegression +from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor, StackingClassifier, StackingRegressor +from sklearn.linear_model import LassoCV, LinearRegression, LogisticRegression, Ridge LearnerInstantiator = Callable[[Dict[str, Any]], Any] # Map learner abbreviations to their instantiation logic @@ -14,6 +15,29 @@ "LGBM Clas.": lambda params: LGBMClassifier(**{**{"verbose": -1, "n_jobs": 1}, **params}), "Linear": lambda params: LinearRegression(**params), "Logistic": lambda params: LogisticRegression(**params), + "Global Linear": lambda params: GlobalRegressor(LinearRegression(**params)), + "Global Logistic": lambda params: GlobalRegressor(LogisticRegression(**params)), + "Stacked Regr.": lambda params: StackingRegressor( + estimators=[ + ("lr", LinearRegression()), + ( + "lgbm", + LGBMRegressor(**{**{"verbose": -1, "n_jobs": 1}, **params}), + ), + ("glr", GlobalRegressor(LinearRegression())), + ], + final_estimator=Ridge(), + ), + "Stacked Clas.": lambda params: StackingClassifier( + estimators=[ + ("lr", LogisticRegression()), + ( + "lgbm", + LGBMClassifier(**{**{"verbose": -1, "n_jobs": 1}, **params}), + ), + ("glr", GlobalRegressor(LogisticRegression())), + ], + ), } From ca45ef9f6514aab0da2ce9d91a9cba2432462c81 Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Mon, 2 Jun 2025 10:55:56 +0200 Subject: [PATCH 03/16] update rdd sharp scripts --- scripts/rdd/rdd_sharp.py | 13 +++++++++ scripts/rdd/rdd_sharp_config.yml | 45 ++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 scripts/rdd/rdd_sharp.py create mode 100644 scripts/rdd/rdd_sharp_config.yml diff --git a/scripts/rdd/rdd_sharp.py b/scripts/rdd/rdd_sharp.py new file mode 100644 index 0000000..3520881 --- /dev/null +++ b/scripts/rdd/rdd_sharp.py @@ -0,0 +1,13 @@ +from montecover.rdd import RDDCoverageSimulation + +# Create and run simulation with config file +sim = RDDCoverageSimulation( + config_file="scripts/rdd/rdd_sharp_config.yml", + log_level="INFO", + log_file="logs/rdd/rdd_sharp_sim.log", +) +sim.run_simulation() +sim.save_results(output_path="results/rdd/", file_prefix="rdd_sharp") + +# Save config file for reproducibility +sim.save_config("results/rdd/rdd_sharp_config.yml") diff --git a/scripts/rdd/rdd_sharp_config.yml b/scripts/rdd/rdd_sharp_config.yml new file mode 100644 index 0000000..2192ac0 --- /dev/null +++ b/scripts/rdd/rdd_sharp_config.yml @@ -0,0 +1,45 @@ +# Simulation parameters for sharp RDD Coverage + +simulation_parameters: + repetitions: 100 + max_runtime: 19800 # 5.5 hours in seconds + random_seed: 42 + n_jobs: -2 + +dgp_parameters: + n_obs: [1000] # Sample size + fuzzy: [False] + cutoff: [0.0] + +# Define reusable learner configurations +learner_definitions: + lgbmr: &lgbmr + name: "LGBM Regr." + params: + n_estimators: 100 + learning_rate: 0.05 + + global_linear: &global_linear + name: "Global Linear" + + local_linear: &local_linear + name: "Linear" + + stacked_reg: &stacked_reg + name: "Stacked Regr." + params: + n_estimators: 100 + learning_rate: 0.05 + +dml_parameters: + fs_specification: ["cutoff", "cutoff and score", "interacted cutoff and score"] + + learners: + - ml_g: *lgbmr + - ml_g: *global_linear + - ml_g: *local_linear + - ml_g: *stacked_reg + + +confidence_parameters: + level: [0.95, 0.90] # Confidence levels From b7d0ffb9bbef2803861ee22a5776cf821948ffd7 Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Mon, 2 Jun 2025 11:04:25 +0200 Subject: [PATCH 04/16] add fuzzy rdd scripts --- scripts/rdd/rdd_fuzzy.py | 13 +++++++ scripts/rdd/rdd_fuzzy_config.yml | 66 ++++++++++++++++++++++++++++++++ scripts/rdd/rdd_sharp_config.yml | 8 ++-- 3 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 scripts/rdd/rdd_fuzzy.py create mode 100644 scripts/rdd/rdd_fuzzy_config.yml diff --git a/scripts/rdd/rdd_fuzzy.py b/scripts/rdd/rdd_fuzzy.py new file mode 100644 index 0000000..3fec90e --- /dev/null +++ b/scripts/rdd/rdd_fuzzy.py @@ -0,0 +1,13 @@ +from montecover.rdd import RDDCoverageSimulation + +# Create and run simulation with config file +sim = RDDCoverageSimulation( + config_file="scripts/rdd/rdd_fuzzy_config.yml", + log_level="INFO", + log_file="logs/rdd/rdd_fuzzy_sim.log", +) +sim.run_simulation() +sim.save_results(output_path="results/rdd/", file_prefix="rdd_fuzzy") + +# Save config file for reproducibility +sim.save_config("results/rdd/rdd_fuzzy_config.yml") diff --git a/scripts/rdd/rdd_fuzzy_config.yml b/scripts/rdd/rdd_fuzzy_config.yml new file mode 100644 index 0000000..830515a --- /dev/null +++ b/scripts/rdd/rdd_fuzzy_config.yml @@ -0,0 +1,66 @@ +# Simulation parameters for fuzzy RDD Coverage + +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 # 5.5 hours in seconds + random_seed: 42 + n_jobs: -2 + +dgp_parameters: + n_obs: [1000] # Sample size + fuzzy: [True] + cutoff: [0.0] + +# Define reusable learner configurations +learner_definitions: + lgbmr: &lgbmr + name: "LGBM Regr." + params: + n_estimators: 100 + learning_rate: 0.05 + + lgbmc: &lgbmc + name: "LGBM Clas." + params: + n_estimators: 100 + learning_rate: 0.05 + + global_linear: &global_linear + name: "Global Linear" + + global_logistic: &global_logistic + name: "Global Logistic" + + local_linear: &local_linear + name: "Linear" + + local_logistic: &local_logistic + name: "Logistic" + + stacked_reg: &stacked_reg + name: "Stacked Regr." + params: + n_estimators: 100 + learning_rate: 0.05 + + stacked_cls: &stacked_cls + name: "Stacked Clas." + params: + n_estimators: 100 + learning_rate: 0.05 + +dml_parameters: + fs_specification: ["cutoff", "cutoff and score", "interacted cutoff and score"] + + learners: + - ml_g: *lgbmr + ml_m: *lgbmc + - ml_g: *global_linear + ml_m: *global_logistic + - ml_g: *local_linear + ml_m: *local_logistic + - ml_g: *stacked_reg + ml_m: *stacked_cls + +confidence_parameters: + level: [0.95, 0.90] # Confidence levels diff --git a/scripts/rdd/rdd_sharp_config.yml b/scripts/rdd/rdd_sharp_config.yml index 2192ac0..560c913 100644 --- a/scripts/rdd/rdd_sharp_config.yml +++ b/scripts/rdd/rdd_sharp_config.yml @@ -1,7 +1,7 @@ # Simulation parameters for sharp RDD Coverage simulation_parameters: - repetitions: 100 + repetitions: 1000 max_runtime: 19800 # 5.5 hours in seconds random_seed: 42 n_jobs: -2 @@ -18,13 +18,13 @@ learner_definitions: params: n_estimators: 100 learning_rate: 0.05 - + global_linear: &global_linear name: "Global Linear" - + local_linear: &local_linear name: "Linear" - + stacked_reg: &stacked_reg name: "Stacked Regr." params: From ae560c7db80ad2fdf8a7ab1a3ad2a4ef1ce5ee12 Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Mon, 2 Jun 2025 11:05:07 +0200 Subject: [PATCH 05/16] add classifier to utils and increase oracle obs --- monte-cover/src/montecover/rdd/rdd.py | 2 +- monte-cover/src/montecover/utils.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/monte-cover/src/montecover/rdd/rdd.py b/monte-cover/src/montecover/rdd/rdd.py index 5faaab7..b01caa0 100644 --- a/monte-cover/src/montecover/rdd/rdd.py +++ b/monte-cover/src/montecover/rdd/rdd.py @@ -51,7 +51,7 @@ def _calculate_oracle_values(self): """Calculate oracle values for the simulation.""" self.logger.info("Calculating oracle values") - data_oracle = make_simple_rdd_data(n_obs=int(1e5), fuzzy=self.fuzzy, cutoff=self.cutoff) + data_oracle = make_simple_rdd_data(n_obs=int(1e6), fuzzy=self.fuzzy, cutoff=self.cutoff) # get oracle value score = data_oracle["score"] ite = data_oracle["oracle_values"]["Y1"] - data_oracle["oracle_values"]["Y0"] diff --git a/monte-cover/src/montecover/utils.py b/monte-cover/src/montecover/utils.py index dfadbbf..838cb43 100644 --- a/monte-cover/src/montecover/utils.py +++ b/monte-cover/src/montecover/utils.py @@ -1,6 +1,6 @@ from typing import Any, Callable, Dict, Tuple -from doubleml.utils import GlobalRegressor +from doubleml.utils import GlobalClassifier, GlobalRegressor from lightgbm import LGBMClassifier, LGBMRegressor from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor, StackingClassifier, StackingRegressor from sklearn.linear_model import LassoCV, LinearRegression, LogisticRegression, Ridge @@ -16,7 +16,7 @@ "Linear": lambda params: LinearRegression(**params), "Logistic": lambda params: LogisticRegression(**params), "Global Linear": lambda params: GlobalRegressor(LinearRegression(**params)), - "Global Logistic": lambda params: GlobalRegressor(LogisticRegression(**params)), + "Global Logistic": lambda params: GlobalClassifier(LogisticRegression(**params)), "Stacked Regr.": lambda params: StackingRegressor( estimators=[ ("lr", LinearRegression()), @@ -35,8 +35,9 @@ "lgbm", LGBMClassifier(**{**{"verbose": -1, "n_jobs": 1}, **params}), ), - ("glr", GlobalRegressor(LogisticRegression())), + ("glr", GlobalClassifier(LogisticRegression())), ], + final_estimator=LogisticRegression(), ), } From ab69ab4ca6b552891115c0356d68f23b10ee746c Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Mon, 2 Jun 2025 12:54:29 +0200 Subject: [PATCH 06/16] update workflows --- .github/workflows/did_sim.yml | 2 +- .github/workflows/rdd_sim.yml | 38 +++++++++++++++++++++-------------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/.github/workflows/did_sim.yml b/.github/workflows/did_sim.yml index 199220d..411502f 100644 --- a/.github/workflows/did_sim.yml +++ b/.github/workflows/did_sim.yml @@ -52,7 +52,7 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v5 with: - version: "0.6.11" + version: "0.7.8" - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/rdd_sim.yml b/.github/workflows/rdd_sim.yml index e79bee9..ea490ee 100644 --- a/.github/workflows/rdd_sim.yml +++ b/.github/workflows/rdd_sim.yml @@ -17,8 +17,8 @@ jobs: strategy: matrix: script: [ - 'scripts/rdd/rdd_sharp_coverage.py', - 'scripts/rdd/rdd_fuzzy_coverage.py', + 'scripts/rdd/rdd_sharp.py', + 'scripts/rdd/rdd_fuzzy.py', ] steps: @@ -48,26 +48,32 @@ jobs: with: ref: ${{ env.TARGET_BRANCH }} + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + version: "0.7.8" + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version-file: "monte-cover/pyproject.toml" - - name: Install dependencies + - name: Install Monte-Cover run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + cd monte-cover + uv venv + uv sync - - name: Install DoubleML from correct branch - run: | - pip uninstall -y doubleml - pip install "doubleml[rdd] @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' - - name: Install RDFlex from main branch + - name: Install DoubleML from correct branch run: | - pip uninstall -y doubleml - pip install git+https://github.com/DoubleML/doubleml-rdflex.git@main - pip install rdrobust + source monte-cover/.venv/bin/activate + uv pip uninstall doubleml + uv pip install "doubleml[rdd] @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}" - name: Set up Git configuration run: | @@ -75,7 +81,9 @@ jobs: git config --global user.email 'github-actions@github.com' - name: Run scripts - run: python ${{ matrix.script }} + run: | + source monte-cover/.venv/bin/activate + uv run ${{ matrix.script }} - name: Commit any existing changes run: | From be7909025c738bcd17152fd3d8cb70428f37a96d Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Mon, 2 Jun 2025 12:54:53 +0200 Subject: [PATCH 07/16] update qmd file --- doc/rdd/rdd.qmd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/rdd/rdd.qmd b/doc/rdd/rdd.qmd index 4a74fb7..dce518d 100644 --- a/doc/rdd/rdd.qmd +++ b/doc/rdd/rdd.qmd @@ -47,7 +47,7 @@ df_sharp = pd.read_csv("../../results/rdd/rdd_sharp_coverage.csv", index_col=Non assert df_sharp["repetition"].nunique() == 1 n_rep_sharp = df_sharp["repetition"].unique()[0] -display_columns_sharp = ["Method", "Learner g", "fs specification", "Bias", "CI Length", "Coverage"] +display_columns_sharp = ["Method", "Learner g", "fs_specification", "Bias", "CI Length", "Coverage"] ``` ```{python} @@ -99,7 +99,7 @@ df_fuzzy = pd.read_csv("../../results/rdd/rdd_fuzzy_coverage.csv", index_col=Non assert df_fuzzy["repetition"].nunique() == 1 n_rep_fuzzy = df_fuzzy["repetition"].unique()[0] -display_columns_fuzzy = ["Method", "Learner g", "Learner m", "fs specification", "Bias", "CI Length", "Coverage"] +display_columns_fuzzy = ["Method", "Learner g", "Learner m", "fs_specification", "Bias", "CI Length", "Coverage"] ``` ```{python} From c4509c9b05e5640249938e561b5961f95c6f1e14 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 2 Jun 2025 13:17:13 +0000 Subject: [PATCH 08/16] Update results from script: scripts/rdd/rdd_sharp.py --- results/rdd/rdd_sharp_config.yml | 41 +++++++++++++++++++++++ results/rdd/rdd_sharp_coverage.csv | 54 +++++++++++++++--------------- results/rdd/rdd_sharp_metadata.csv | 2 ++ 3 files changed, 70 insertions(+), 27 deletions(-) create mode 100644 results/rdd/rdd_sharp_config.yml create mode 100644 results/rdd/rdd_sharp_metadata.csv diff --git a/results/rdd/rdd_sharp_config.yml b/results/rdd/rdd_sharp_config.yml new file mode 100644 index 0000000..57d0a43 --- /dev/null +++ b/results/rdd/rdd_sharp_config.yml @@ -0,0 +1,41 @@ +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + n_obs: + - 1000 + fuzzy: + - false + cutoff: + - 0.0 +learner_definitions: + lgbmr: &id001 + name: LGBM Regr. + params: + n_estimators: 100 + learning_rate: 0.05 + global_linear: &id002 + name: Global Linear + local_linear: &id003 + name: Linear + stacked_reg: &id004 + name: Stacked Regr. + params: + n_estimators: 100 + learning_rate: 0.05 +dml_parameters: + fs_specification: + - cutoff + - cutoff and score + - interacted cutoff and score + learners: + - ml_g: *id001 + - ml_g: *id002 + - ml_g: *id003 + - ml_g: *id004 +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/rdd/rdd_sharp_coverage.csv b/results/rdd/rdd_sharp_coverage.csv index 558f06e..f9b942e 100644 --- a/results/rdd/rdd_sharp_coverage.csv +++ b/results/rdd/rdd_sharp_coverage.csv @@ -1,27 +1,27 @@ -Method,fs specification,Learner g,level,Coverage,CI Length,Bias,repetition -rdflex,cutoff,Global linear,0.9,0.874,2.2011088216647567,0.5534513823633507,500 -rdflex,cutoff,Global linear,0.95,0.922,2.6227829308507813,0.5534513823633507,500 -rdflex,cutoff,LGBM,0.9,0.914,0.5720094363002372,0.1386055523323647,500 -rdflex,cutoff,LGBM,0.95,0.96,0.681591283014874,0.1386055523323647,500 -rdflex,cutoff,Linear,0.9,0.874,2.2135060296971463,0.558126057195101,500 -rdflex,cutoff,Linear,0.95,0.918,2.6375551153504855,0.558126057195101,500 -rdflex,cutoff,Stacked,0.9,0.9,0.5590842827226126,0.12977180379581715,500 -rdflex,cutoff,Stacked,0.95,0.964,0.6661900125968221,0.12977180379581715,500 -rdflex,cutoff and score,Global linear,0.9,0.876,2.2005650015589864,0.5551812535814453,500 -rdflex,cutoff and score,Global linear,0.95,0.922,2.622134929226859,0.5551812535814453,500 -rdflex,cutoff and score,LGBM,0.9,0.902,0.5984294788353389,0.14476299319184177,500 -rdflex,cutoff and score,LGBM,0.95,0.95,0.7130727054286046,0.14476299319184177,500 -rdflex,cutoff and score,Linear,0.9,0.87,2.212887444081595,0.5582781571131341,500 -rdflex,cutoff and score,Linear,0.95,0.922,2.63681802512679,0.5582781571131341,500 -rdflex,cutoff and score,Stacked,0.9,0.88,0.5820101876494705,0.14231610717047102,500 -rdflex,cutoff and score,Stacked,0.95,0.956,0.6935079132497274,0.14231610717047102,500 -rdflex,interacted cutoff and score,Global linear,0.9,0.878,2.202268208292408,0.5546663773642981,500 -rdflex,interacted cutoff and score,Global linear,0.95,0.926,2.6241644252264025,0.5546663773642981,500 -rdflex,interacted cutoff and score,LGBM,0.9,0.886,0.6002731299225453,0.151487055237104,500 -rdflex,interacted cutoff and score,LGBM,0.95,0.948,0.7152695511975984,0.151487055237104,500 -rdflex,interacted cutoff and score,Linear,0.9,0.88,2.2252512728137637,0.5541600195953326,500 -rdflex,interacted cutoff and score,Linear,0.95,0.916,2.651550435737079,0.5541600195953326,500 -rdflex,interacted cutoff and score,Stacked,0.9,0.904,0.5793073094025809,0.14171607962579613,500 -rdflex,interacted cutoff and score,Stacked,0.95,0.962,0.6902872351713266,0.14171607962579613,500 -rdrobust,cutoff,linear,0.9,0.874,2.1797037623552287,0.555486091306879,500 -rdrobust,cutoff,linear,0.95,0.916,2.597277229525019,0.555486091306879,500 +Method,fs_specification,Learner g,Learner m,level,Coverage,CI Length,Bias,repetition +RDFlex,cutoff,Global Linear,N/A,0.9,0.8783333333333334,1.9718196650713942,0.5093136469698075,1000 +RDFlex,cutoff,Global Linear,N/A,0.95,0.9376666666666666,2.3495680492315225,0.5093136469698075,1000 +RDFlex,cutoff,LGBM Regr.,N/A,0.9,0.8753333333333334,0.5757111703216161,0.15519089437437322,1000 +RDFlex,cutoff,LGBM Regr.,N/A,0.95,0.9316666666666666,0.6860021711591865,0.15519089437437322,1000 +RDFlex,cutoff,Linear,N/A,0.9,0.8823333333333334,1.9835872651996145,0.5100383308045482,1000 +RDFlex,cutoff,Linear,N/A,0.95,0.942,2.363590009640563,0.5100383308045482,1000 +RDFlex,cutoff,Stacked Regr.,N/A,0.9,0.882,0.5590578781507813,0.14332464663035524,1000 +RDFlex,cutoff,Stacked Regr.,N/A,0.95,0.9366666666666666,0.6661585496088891,0.14332464663035524,1000 +RDFlex,cutoff and score,Global Linear,N/A,0.9,0.882,1.971849810438757,0.5080389655599933,1000 +RDFlex,cutoff and score,Global Linear,N/A,0.95,0.9383333333333334,2.34960396965226,0.5080389655599933,1000 +RDFlex,cutoff and score,LGBM Regr.,N/A,0.9,0.8846666666666666,0.6017477258893029,0.15725459328740565,1000 +RDFlex,cutoff and score,LGBM Regr.,N/A,0.95,0.9423333333333334,0.7170266406669815,0.15725459328740565,1000 +RDFlex,cutoff and score,Linear,N/A,0.9,0.8826666666666666,1.9835459135793096,0.5104954020103588,1000 +RDFlex,cutoff and score,Linear,N/A,0.95,0.9403333333333334,2.363540736145845,0.5104954020103588,1000 +RDFlex,cutoff and score,Stacked Regr.,N/A,0.9,0.8973333333333333,0.5804002753516253,0.14754701246061155,1000 +RDFlex,cutoff and score,Stacked Regr.,N/A,0.95,0.9413333333333334,0.6915895844268198,0.14754701246061155,1000 +RDFlex,interacted cutoff and score,Global Linear,N/A,0.9,0.882,1.9748055181211774,0.508213715385231,1000 +RDFlex,interacted cutoff and score,Global Linear,N/A,0.95,0.938,2.3531259125847197,0.508213715385231,1000 +RDFlex,interacted cutoff and score,LGBM Regr.,N/A,0.9,0.889,0.6013814494909888,0.15466636196782954,1000 +RDFlex,interacted cutoff and score,LGBM Regr.,N/A,0.95,0.9433333333333334,0.7165901954190156,0.15466636196782954,1000 +RDFlex,interacted cutoff and score,Linear,N/A,0.9,0.88,1.992308168049787,0.5113923914828905,1000 +RDFlex,interacted cutoff and score,Linear,N/A,0.95,0.938,2.373981606326701,0.5113923914828905,1000 +RDFlex,interacted cutoff and score,Stacked Regr.,N/A,0.9,0.8756666666666666,0.5801425055800902,0.14962735197253088,1000 +RDFlex,interacted cutoff and score,Stacked Regr.,N/A,0.95,0.929,0.6912824327993231,0.14962735197253088,1000 +rdrobust,cutoff,Linear,Logistic,0.9,0.896,2.178474435204289,0.5326223192522394,1000 +rdrobust,cutoff,Linear,Logistic,0.95,0.954,2.595812395875642,0.5326223192522394,1000 diff --git a/results/rdd/rdd_sharp_metadata.csv b/results/rdd/rdd_sharp_metadata.csv new file mode 100644 index 0000000..375d001 --- /dev/null +++ b/results/rdd/rdd_sharp_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,RDDCoverageSimulation,2025-06-02 13:17,65.69356084664663,3.12.3,scripts/rdd/rdd_sharp_config.yml From 85d424c6efdae7d3daa6654ce9982c18773e8a59 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 2 Jun 2025 14:19:40 +0000 Subject: [PATCH 09/16] Update results from script: scripts/rdd/rdd_fuzzy.py --- results/rdd/rdd_fuzzy_config.yml | 59 ++++++++++++++ results/rdd/rdd_fuzzy_coverage.csv | 126 +++++++---------------------- results/rdd/rdd_fuzzy_metadata.csv | 2 + 3 files changed, 88 insertions(+), 99 deletions(-) create mode 100644 results/rdd/rdd_fuzzy_config.yml create mode 100644 results/rdd/rdd_fuzzy_metadata.csv diff --git a/results/rdd/rdd_fuzzy_config.yml b/results/rdd/rdd_fuzzy_config.yml new file mode 100644 index 0000000..d8b09c0 --- /dev/null +++ b/results/rdd/rdd_fuzzy_config.yml @@ -0,0 +1,59 @@ +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + n_obs: + - 1000 + fuzzy: + - true + cutoff: + - 0.0 +learner_definitions: + lgbmr: &id001 + name: LGBM Regr. + params: + n_estimators: 100 + learning_rate: 0.05 + lgbmc: &id002 + name: LGBM Clas. + params: + n_estimators: 100 + learning_rate: 0.05 + global_linear: &id003 + name: Global Linear + global_logistic: &id004 + name: Global Logistic + local_linear: &id005 + name: Linear + local_logistic: &id006 + name: Logistic + stacked_reg: &id007 + name: Stacked Regr. + params: + n_estimators: 100 + learning_rate: 0.05 + stacked_cls: &id008 + name: Stacked Clas. + params: + n_estimators: 100 + learning_rate: 0.05 +dml_parameters: + fs_specification: + - cutoff + - cutoff and score + - interacted cutoff and score + learners: + - ml_g: *id001 + ml_m: *id002 + - ml_g: *id003 + ml_m: *id004 + - ml_g: *id005 + ml_m: *id006 + - ml_g: *id007 + ml_m: *id008 +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/rdd/rdd_fuzzy_coverage.csv b/results/rdd/rdd_fuzzy_coverage.csv index ebec18a..71a7b18 100644 --- a/results/rdd/rdd_fuzzy_coverage.csv +++ b/results/rdd/rdd_fuzzy_coverage.csv @@ -1,99 +1,27 @@ -Method,fs specification,Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -rdflex,cutoff,Global linear,Global linear,0.9,0.928,10.556606789284906,2.56683549934152,250 -rdflex,cutoff,Global linear,Global linear,0.95,0.964,12.578972844104541,2.56683549934152,250 -rdflex,cutoff,Global linear,LGBM,0.9,0.932,10.775966548524359,2.6575671123131777,250 -rdflex,cutoff,Global linear,LGBM,0.95,0.976,12.840356119018518,2.6575671123131777,250 -rdflex,cutoff,Global linear,Linear,0.9,0.928,10.680654195730742,2.6095109173162823,250 -rdflex,cutoff,Global linear,Linear,0.95,0.964,12.726784445711942,2.6095109173162823,250 -rdflex,cutoff,Global linear,Stacked,0.9,0.932,10.571620600845328,2.6448751082127377,250 -rdflex,cutoff,Global linear,Stacked,0.95,0.96,12.596862904014394,2.6448751082127377,250 -rdflex,cutoff,LGBM,Global linear,0.9,0.94,2.038612360942403,0.46302745914972104,250 -rdflex,cutoff,LGBM,Global linear,0.95,0.968,2.4291564552711176,0.46302745914972104,250 -rdflex,cutoff,LGBM,LGBM,0.9,0.94,2.0417477871653618,0.4912050569704094,250 -rdflex,cutoff,LGBM,LGBM,0.95,0.976,2.4328925460529893,0.4912050569704094,250 -rdflex,cutoff,LGBM,Linear,0.9,0.96,2.088603350877394,0.47982216259182514,250 -rdflex,cutoff,LGBM,Linear,0.95,0.992,2.488724393851574,0.47982216259182514,250 -rdflex,cutoff,LGBM,Stacked,0.9,0.916,2.0184810970572267,0.46215722170125356,250 -rdflex,cutoff,LGBM,Stacked,0.95,0.972,2.405168574810687,0.46215722170125356,250 -rdflex,cutoff,Linear,Global linear,0.9,0.928,10.564695732971051,2.553945752977564,250 -rdflex,cutoff,Linear,Global linear,0.95,0.968,12.58861141500109,2.553945752977564,250 -rdflex,cutoff,Linear,LGBM,0.9,0.928,10.777243710702589,2.650598511347432,250 -rdflex,cutoff,Linear,LGBM,0.95,0.972,12.841877951618534,2.650598511347432,250 -rdflex,cutoff,Linear,Linear,0.9,0.92,10.74105034617443,2.6072040698667376,250 -rdflex,cutoff,Linear,Linear,0.95,0.968,12.798750897762686,2.6072040698667376,250 -rdflex,cutoff,Linear,Stacked,0.9,0.932,10.404127851046006,2.505787762202565,250 -rdflex,cutoff,Linear,Stacked,0.95,0.968,12.397282982798743,2.505787762202565,250 -rdflex,cutoff,Stacked,Global linear,0.9,0.928,2.100685938954007,0.4945555538677959,250 -rdflex,cutoff,Stacked,Global linear,0.95,0.972,2.50312168555107,0.4945555538677959,250 -rdflex,cutoff,Stacked,LGBM,0.9,0.956,2.003482953646103,0.4641511546138578,250 -rdflex,cutoff,Stacked,LGBM,0.95,0.992,2.3872971846522506,0.4641511546138578,250 -rdflex,cutoff,Stacked,Linear,0.9,0.944,2.11736656727605,0.46071259221608185,250 -rdflex,cutoff,Stacked,Linear,0.95,0.984,2.522997880134595,0.46071259221608185,250 -rdflex,cutoff,Stacked,Stacked,0.9,0.936,2.0245073903258874,0.44210382922098473,250 -rdflex,cutoff,Stacked,Stacked,0.95,0.976,2.412349346140925,0.44210382922098473,250 -rdflex,cutoff and score,Global linear,Global linear,0.9,0.928,10.544420005750476,2.567803474303041,250 -rdflex,cutoff and score,Global linear,Global linear,0.95,0.964,12.564451395859244,2.567803474303041,250 -rdflex,cutoff and score,Global linear,LGBM,0.9,0.928,10.805365747954351,2.6437437844814258,250 -rdflex,cutoff and score,Global linear,LGBM,0.95,0.972,12.875387425824758,2.6437437844814258,250 -rdflex,cutoff and score,Global linear,Linear,0.9,0.928,10.699263774918355,2.5999147658504103,250 -rdflex,cutoff and score,Global linear,Linear,0.95,0.964,12.748959127019463,2.5999147658504103,250 -rdflex,cutoff and score,Global linear,Stacked,0.9,0.932,10.82713502969946,2.6270982998447745,250 -rdflex,cutoff and score,Global linear,Stacked,0.95,0.972,12.901327124950933,2.6270982998447745,250 -rdflex,cutoff and score,LGBM,Global linear,0.9,0.956,2.1656416896389774,0.4963137476876054,250 -rdflex,cutoff and score,LGBM,Global linear,0.95,0.984,2.5805212363957613,0.4963137476876054,250 -rdflex,cutoff and score,LGBM,LGBM,0.9,0.94,2.201449781464929,0.5341193634721544,250 -rdflex,cutoff and score,LGBM,LGBM,0.95,0.972,2.6231892095114255,0.5341193634721544,250 -rdflex,cutoff and score,LGBM,Linear,0.9,0.952,2.1894694844073546,0.4635302319336836,250 -rdflex,cutoff and score,LGBM,Linear,0.95,0.98,2.6089138050789615,0.4635302319336836,250 -rdflex,cutoff and score,LGBM,Stacked,0.9,0.964,2.1361906429140674,0.5017184797294254,250 -rdflex,cutoff and score,LGBM,Stacked,0.95,0.984,2.545428149727124,0.5017184797294254,250 -rdflex,cutoff and score,Linear,Global linear,0.9,0.92,10.636843214150689,2.585401553398786,250 -rdflex,cutoff and score,Linear,Global linear,0.95,0.968,12.67458044128427,2.585401553398786,250 -rdflex,cutoff and score,Linear,LGBM,0.9,0.936,10.869584147939532,2.6903868750455233,250 -rdflex,cutoff and score,Linear,LGBM,0.95,0.98,12.95190836911928,2.6903868750455233,250 -rdflex,cutoff and score,Linear,Linear,0.9,0.912,10.715620665219745,2.5870008277605034,250 -rdflex,cutoff and score,Linear,Linear,0.95,0.968,12.76844956395835,2.5870008277605034,250 -rdflex,cutoff and score,Linear,Stacked,0.9,0.92,10.814063241507966,2.6455094849616714,250 -rdflex,cutoff and score,Linear,Stacked,0.95,0.964,12.88575112861359,2.6455094849616714,250 -rdflex,cutoff and score,Stacked,Global linear,0.9,0.932,2.1726374228239367,0.4855323537680247,250 -rdflex,cutoff and score,Stacked,Global linear,0.95,0.976,2.5888571666349667,0.4855323537680247,250 -rdflex,cutoff and score,Stacked,LGBM,0.9,0.952,2.14733586500187,0.5004783937020142,250 -rdflex,cutoff and score,Stacked,LGBM,0.95,0.972,2.5587085009595185,0.5004783937020142,250 -rdflex,cutoff and score,Stacked,Linear,0.9,0.948,2.202424304512329,0.49388698262577957,250 -rdflex,cutoff and score,Stacked,Linear,0.95,0.992,2.6243504253446837,0.49388698262577957,250 -rdflex,cutoff and score,Stacked,Stacked,0.9,0.956,2.23318991538815,0.4972797313578032,250 -rdflex,cutoff and score,Stacked,Stacked,0.95,0.98,2.6610099118126316,0.4972797313578032,250 -rdflex,interacted cutoff and score,Global linear,Global linear,0.9,0.932,10.545804244171494,2.539582402518461,250 -rdflex,interacted cutoff and score,Global linear,Global linear,0.95,0.964,12.566100817672075,2.539582402518461,250 -rdflex,interacted cutoff and score,Global linear,LGBM,0.9,0.932,10.884267411835014,2.650211237977455,250 -rdflex,interacted cutoff and score,Global linear,LGBM,0.95,0.972,12.969404557192851,2.650211237977455,250 -rdflex,interacted cutoff and score,Global linear,Linear,0.9,0.936,10.674123116042953,2.5790433126760175,250 -rdflex,interacted cutoff and score,Global linear,Linear,0.95,0.964,12.719002184264186,2.5790433126760175,250 -rdflex,interacted cutoff and score,Global linear,Stacked,0.9,0.932,10.626731756603398,2.6159768660130136,250 -rdflex,interacted cutoff and score,Global linear,Stacked,0.95,0.968,12.662531896478109,2.6159768660130136,250 -rdflex,interacted cutoff and score,LGBM,Global linear,0.9,0.936,2.1565580135903732,0.5032955415920486,250 -rdflex,interacted cutoff and score,LGBM,Global linear,0.95,0.98,2.569697368781784,0.5032955415920486,250 -rdflex,interacted cutoff and score,LGBM,LGBM,0.9,0.944,2.2329626116556094,0.5196735971141546,250 -rdflex,interacted cutoff and score,LGBM,LGBM,0.95,0.98,2.6607390627096894,0.5196735971141546,250 -rdflex,interacted cutoff and score,LGBM,Linear,0.9,0.94,2.188972984996604,0.5079568034409114,250 -rdflex,interacted cutoff and score,LGBM,Linear,0.95,0.988,2.608322189540976,0.5079568034409114,250 -rdflex,interacted cutoff and score,LGBM,Stacked,0.9,0.94,2.1435376000141364,0.5369566034254465,250 -rdflex,interacted cutoff and score,LGBM,Stacked,0.95,0.988,2.55418258907428,0.5369566034254465,250 -rdflex,interacted cutoff and score,Linear,Global linear,0.9,0.92,10.689912181088147,2.611324508403497,250 -rdflex,interacted cutoff and score,Linear,Global linear,0.95,0.968,12.737816015678167,2.611324508403497,250 -rdflex,interacted cutoff and score,Linear,LGBM,0.9,0.936,10.997728797394373,2.7029090494109984,250 -rdflex,interacted cutoff and score,Linear,LGBM,0.95,0.976,13.104602136897563,2.7029090494109984,250 -rdflex,interacted cutoff and score,Linear,Linear,0.9,0.936,10.779112620544133,2.610958342807748,250 -rdflex,interacted cutoff and score,Linear,Linear,0.95,0.968,12.844104895049702,2.610958342807748,250 -rdflex,interacted cutoff and score,Linear,Stacked,0.9,0.92,10.737943290197988,2.6468998393072347,250 -rdflex,interacted cutoff and score,Linear,Stacked,0.95,0.968,12.795048612214599,2.6468998393072347,250 -rdflex,interacted cutoff and score,Stacked,Global linear,0.9,0.932,2.233890329611388,0.5262031584833204,250 -rdflex,interacted cutoff and score,Stacked,Global linear,0.95,0.976,2.661844506836355,0.5262031584833204,250 -rdflex,interacted cutoff and score,Stacked,LGBM,0.9,0.936,2.20946398801941,0.5095892633671805,250 -rdflex,interacted cutoff and score,Stacked,LGBM,0.95,0.98,2.632738725622813,0.5095892633671805,250 -rdflex,interacted cutoff and score,Stacked,Linear,0.9,0.928,2.2332457187032886,0.49902647466338274,250 -rdflex,interacted cutoff and score,Stacked,Linear,0.95,0.968,2.661076405563868,0.49902647466338274,250 -rdflex,interacted cutoff and score,Stacked,Stacked,0.9,0.948,2.181302390767035,0.4919155381101698,250 -rdflex,interacted cutoff and score,Stacked,Stacked,0.95,0.984,2.599182112768406,0.4919155381101698,250 -rdrobust,cutoff,linear,linear,0.9,0.928,10.396400783944904,2.560336967798922,250 -rdrobust,cutoff,linear,linear,0.95,0.964,12.388075614449283,2.560336967798922,250 +Method,fs_specification,Learner g,Learner m,level,Coverage,CI Length,Bias,repetition +RDFlex,cutoff,Global Linear,Global Logistic,0.9,0.9363333333333334,61.21542033320797,6.094527516865915,1000 +RDFlex,cutoff,Global Linear,Global Logistic,0.95,0.972,72.942672336101,6.094527516865915,1000 +RDFlex,cutoff,LGBM Regr.,LGBM Clas.,0.9,0.9513333333333334,8.933409181593555,1.5599808778760562,1000 +RDFlex,cutoff,LGBM Regr.,LGBM Clas.,0.95,0.9876666666666666,10.644813598115608,1.5599808778760562,1000 +RDFlex,cutoff,Linear,Logistic,0.9,0.939,34.93607282259327,5.6843493908942815,1000 +RDFlex,cutoff,Linear,Logistic,0.95,0.973,41.62889838438588,5.6843493908942815,1000 +RDFlex,cutoff,Stacked Regr.,Stacked Clas.,0.9,0.9503333333333334,6.7442427655640484,1.3456834189094324,1000 +RDFlex,cutoff,Stacked Regr.,Stacked Clas.,0.95,0.9853333333333334,8.036260921282771,1.3456834189094324,1000 +RDFlex,cutoff and score,Global Linear,Global Logistic,0.9,0.9346666666666666,4016.9877724310254,50.8166330459716,1000 +RDFlex,cutoff and score,Global Linear,Global Logistic,0.95,0.9713333333333334,4786.536158171398,50.8166330459716,1000 +RDFlex,cutoff and score,LGBM Regr.,LGBM Clas.,0.9,0.951,9.065887957351245,1.5153501497320774,1000 +RDFlex,cutoff and score,LGBM Regr.,LGBM Clas.,0.95,0.9873333333333334,10.802671795919059,1.5153501497320774,1000 +RDFlex,cutoff and score,Linear,Logistic,0.9,0.94,335.8084886364535,7.708410484814828,1000 +RDFlex,cutoff and score,Linear,Logistic,0.95,0.9746666666666667,400.1404943551824,7.708410484814828,1000 +RDFlex,cutoff and score,Stacked Regr.,Stacked Clas.,0.9,0.9496666666666667,7.718643198281012,1.36917558119484,1000 +RDFlex,cutoff and score,Stacked Regr.,Stacked Clas.,0.95,0.9866666666666666,9.1973306501346,1.36917558119484,1000 +RDFlex,interacted cutoff and score,Global Linear,Global Logistic,0.9,0.939,49.6899424879847,5.516111563740509,1000 +RDFlex,interacted cutoff and score,Global Linear,Global Logistic,0.95,0.9723333333333334,59.20921842195623,5.516111563740509,1000 +RDFlex,interacted cutoff and score,LGBM Regr.,LGBM Clas.,0.9,0.949,8.869451650244981,1.5644296749591,1000 +RDFlex,interacted cutoff and score,LGBM Regr.,LGBM Clas.,0.95,0.9866666666666666,10.568603498974065,1.5644296749591,1000 +RDFlex,interacted cutoff and score,Linear,Logistic,0.9,0.939,165.6185238963947,6.482221436992595,1000 +RDFlex,interacted cutoff and score,Linear,Logistic,0.95,0.9733333333333334,197.34664330663676,6.482221436992595,1000 +RDFlex,interacted cutoff and score,Stacked Regr.,Stacked Clas.,0.9,0.953,12.965304115439578,1.8427674267462426,1000 +RDFlex,interacted cutoff and score,Stacked Regr.,Stacked Clas.,0.95,0.9833333333333334,15.449112734710363,1.8427674267462426,1000 +rdrobust,cutoff,Linear,Logistic,0.9,0.66,2.2540261137473974,0.9256316634723921,1000 +rdrobust,cutoff,Linear,Logistic,0.95,0.768,2.685837773507866,0.9256316634723921,1000 diff --git a/results/rdd/rdd_fuzzy_metadata.csv b/results/rdd/rdd_fuzzy_metadata.csv new file mode 100644 index 0000000..1bee153 --- /dev/null +++ b/results/rdd/rdd_fuzzy_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,RDDCoverageSimulation,2025-06-02 14:19,128.11025975545246,3.12.3,scripts/rdd/rdd_fuzzy_config.yml From e952f83782b95649757749e9cf280f00b04f7acd Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 2 Jun 2025 15:10:52 +0000 Subject: [PATCH 10/16] Update results from script: scripts/did/did_pa_atte_coverage.py --- results/did/did_pa_atte_coverage_metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/did/did_pa_atte_coverage_metadata.csv b/results/did/did_pa_atte_coverage_metadata.csv index 5a18477..961444b 100644 --- a/results/did/did_pa_atte_coverage_metadata.csv +++ b/results/did/did_pa_atte_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,did_pa_atte_coverage.py,2025-05-22 14:30:49,10714.369587659836,3.12.3 +0.11.dev0,did_pa_atte_coverage.py,2025-06-02 15:10:48,10769.977479457855,3.12.3 From de82da6ea7223b782cbe2af236e0a68f615e3b5c Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 2 Jun 2025 15:42:32 +0000 Subject: [PATCH 11/16] Update results from script: scripts/did/did_cs_atte_coverage.py --- results/did/did_cs_atte_coverage_metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/did/did_cs_atte_coverage_metadata.csv b/results/did/did_cs_atte_coverage_metadata.csv index 19aa007..c055b9e 100644 --- a/results/did/did_cs_atte_coverage_metadata.csv +++ b/results/did/did_cs_atte_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,did_cs_atte_coverage.py,2025-05-22 15:05:03,12767.107241630554,3.12.3 +0.11.dev0,did_cs_atte_coverage.py,2025-06-02 15:42:26,12669.156663179398,3.12.3 From bd4246fc4641ed87537129f78a9a45f92055ad03 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 2 Jun 2025 17:51:11 +0000 Subject: [PATCH 12/16] Update results from script: scripts/did/did_pa_multi.py --- results/did/did_multi_detailed.csv | 96 +++++++++--------- results/did/did_multi_eventstudy.csv | 96 +++++++++--------- results/did/did_multi_group.csv | 96 +++++++++--------- results/did/did_multi_metadata.csv | 2 +- results/did/did_multi_time.csv | 96 +++++++++--------- results/did/did_pa_multi_config.yml | 140 ++++++--------------------- 6 files changed, 223 insertions(+), 303 deletions(-) diff --git a/results/did/did_multi_detailed.csv b/results/did/did_multi_detailed.csv index 3384909..0792d64 100644 --- a/results/did/did_multi_detailed.csv +++ b/results/did/did_multi_detailed.csv @@ -1,49 +1,49 @@ Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,experimental,False,1,0.9,0.4084166666666667,0.6711379447655711,0.45573140858217215,0.079,1.0050739084247675,1000 -LGBM,LGBM,experimental,False,1,0.95,0.49625,0.7997101862715227,0.45573140858217215,0.132,1.115181838446232,1000 -LGBM,LGBM,experimental,False,4,0.9,0.5408333333333334,0.5829183572257842,0.3256247127712507,0.227,0.8979506620328224,1000 -LGBM,LGBM,experimental,False,4,0.95,0.62975,0.6945900640455575,0.3256247127712507,0.299,0.9884673580918313,1000 -LGBM,LGBM,experimental,False,6,0.9,0.8979166666666666,0.5800302078090503,0.14243775358828042,0.89,0.892704703388473,1000 -LGBM,LGBM,experimental,False,6,0.95,0.95,0.691148621751838,0.14243775358828042,0.955,0.9829906161123636,1000 -LGBM,LGBM,experimental,True,1,0.9,0.4105833333333333,0.671233414997924,0.45509646764918477,0.081,1.005300983475438,1000 -LGBM,LGBM,experimental,True,1,0.95,0.4990833333333333,0.7998239460699273,0.45509646764918477,0.139,1.1154305467306445,1000 -LGBM,LGBM,experimental,True,4,0.9,0.5370833333333334,0.5828484393127565,0.3258110613161524,0.212,0.8973624125797166,1000 -LGBM,LGBM,experimental,True,4,0.95,0.6318333333333334,0.6945067517135889,0.3258110613161524,0.305,0.9879336259779586,1000 -LGBM,LGBM,experimental,True,6,0.9,0.89625,0.5799071420563836,0.14138962778278252,0.903,0.8926727198727438,1000 -LGBM,LGBM,experimental,True,6,0.95,0.947,0.6910019798628547,0.14138962778278252,0.954,0.982776272537425,1000 -LGBM,LGBM,observational,False,1,0.9,0.90725,2.7320711716537267,0.7081787206178205,0.946,4.2481549947682335,1000 -LGBM,LGBM,observational,False,1,0.95,0.9646666666666667,3.255463593782398,0.7081787206178205,0.985,4.664072261741128,1000 -LGBM,LGBM,observational,False,4,0.9,0.9076666666666666,3.5140296061122283,0.9722309202462336,0.972,5.406999616722633,1000 -LGBM,LGBM,observational,False,4,0.95,0.9648333333333333,4.187224538241929,0.9722309202462336,0.995,5.954805878504198,1000 -LGBM,LGBM,observational,False,6,0.9,0.92525,2.166594877755592,0.5140262158838963,0.96,3.3799046023620645,1000 -LGBM,LGBM,observational,False,6,0.95,0.9675833333333334,2.5816570300909847,0.5140262158838963,0.984,3.7082268657228123,1000 -LGBM,LGBM,observational,True,1,0.9,0.9088333333333334,1.1285770515711326,0.2779542066229801,0.934,1.7607428145840895,1000 -LGBM,LGBM,observational,True,1,0.95,0.9595833333333333,1.3447825013812504,0.2779542066229801,0.971,1.9310262593666496,1000 -LGBM,LGBM,observational,True,4,0.9,0.92175,1.4119321412730104,0.3268873239533469,0.941,2.1833548301330525,1000 -LGBM,LGBM,observational,True,4,0.95,0.9650833333333334,1.6824209158589551,0.3268873239533469,0.975,2.400470983600388,1000 -LGBM,LGBM,observational,True,6,0.9,0.9054166666666666,1.0205289656802177,0.2486678941122539,0.917,1.597156856017389,1000 -LGBM,LGBM,observational,True,6,0.95,0.9546666666666667,1.2160352660803362,0.2486678941122539,0.957,1.7513439251743073,1000 -Linear,Linear,experimental,False,1,0.9,0.84575,0.2947177158765365,0.08161611344312397,0.752,0.4590941708451621,1000 -Linear,Linear,experimental,False,1,0.95,0.9100833333333334,0.35117781865763664,0.08161611344312397,0.853,0.504099184053661,1000 -Linear,Linear,experimental,False,4,0.9,0.3073333333333333,0.974825708310707,0.808795845992618,0.033,1.4112500397019019,1000 -Linear,Linear,experimental,False,4,0.95,0.38408333333333333,1.1615764759772769,0.808795845992618,0.069,1.5731674507528597,1000 -Linear,Linear,experimental,False,6,0.9,0.8911666666666667,0.9832819402273247,0.243739180309483,0.893,1.4205928620859418,1000 -Linear,Linear,experimental,False,6,0.95,0.9423333333333334,1.171652697794173,0.243739180309483,0.949,1.5853446204128676,1000 -Linear,Linear,experimental,True,1,0.9,0.8463333333333334,0.2947197524013989,0.08159475357883039,0.759,0.4593380570357237,1000 -Linear,Linear,experimental,True,1,0.95,0.9099166666666666,0.351180245326684,0.08159475357883039,0.86,0.5045635273614648,1000 -Linear,Linear,experimental,True,4,0.9,0.30625,0.9748445034208943,0.8085550484761943,0.034,1.4110989059663714,1000 -Linear,Linear,experimental,True,4,0.95,0.38475,1.1615988717324064,0.8085550484761943,0.068,1.574069732455078,1000 -Linear,Linear,experimental,True,6,0.9,0.89,0.9832818903405501,0.24368928454372696,0.893,1.4193554640817743,1000 -Linear,Linear,experimental,True,6,0.95,0.94225,1.1716526383504147,0.24368928454372696,0.952,1.5837169582518265,1000 -Linear,Linear,observational,False,1,0.9,0.9005,0.3188204993145348,0.07710334427560887,0.894,0.495905282385802,1000 -Linear,Linear,observational,False,1,0.95,0.9494166666666666,0.37989805655090114,0.07710334427560887,0.948,0.5446250222430568,1000 -Linear,Linear,observational,False,4,0.9,0.42083333333333334,1.2366389453819784,0.7873737934686624,0.183,1.7673158966945293,1000 -Linear,Linear,observational,False,4,0.95,0.527,1.4735461898335716,0.7873737934686624,0.272,1.977244843861637,1000 -Linear,Linear,observational,False,6,0.9,0.8901666666666667,1.0315660555384851,0.2592484851241425,0.889,1.4876338955955155,1000 -Linear,Linear,observational,False,6,0.95,0.9431666666666666,1.2291867698140935,0.2592484851241425,0.937,1.6596741796186278,1000 -Linear,Linear,observational,True,1,0.9,0.8986666666666666,0.31665078693066356,0.07694111946321994,0.883,0.49251977204037817,1000 -Linear,Linear,observational,True,1,0.95,0.9495,0.37731268478315316,0.07694111946321994,0.944,0.5411752370665596,1000 -Linear,Linear,observational,True,4,0.9,0.417,1.2352689208194765,0.7872341789805914,0.183,1.7660298619152044,1000 -Linear,Linear,observational,True,4,0.95,0.5238333333333334,1.4719137048778035,0.7872341789805914,0.278,1.9751958823009785,1000 -Linear,Linear,observational,True,6,0.9,0.8873333333333334,1.023987089801951,0.259336442252307,0.88,1.4789961469734276,1000 -Linear,Linear,observational,True,6,0.95,0.9425,1.2201558751251838,0.259336442252307,0.947,1.6504862891485015,1000 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.4084507042253521,0.6612522092326544,0.44973970228836385,0.10328638497652583,0.9935059914244001,213 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.49178403755868544,0.787930605834864,0.44973970228836385,0.1596244131455399,1.1014902175259917,213 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.5336463223787168,0.582766759983815,0.32299857116015934,0.18309859154929578,0.8961664076512703,213 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.6263693270735524,0.694409424790157,0.32299857116015934,0.28169014084507044,0.9867179947950292,213 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.9017996870109547,0.578959671088662,0.13751972629200543,0.8732394366197183,0.8919682440411882,213 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9491392801251956,0.6898729985707177,0.13751972629200543,0.9342723004694836,0.9817237170113741,213 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.4107981220657277,0.6610990972056453,0.4476286561990305,0.107981220657277,0.9924537064343187,213 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.5050860719874805,0.7877481616017598,0.4476286561990305,0.1643192488262911,1.099584135360425,213 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.5348200312989045,0.5827020149697208,0.32210086748537925,0.20187793427230047,0.8980725212627803,213 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.6255868544600939,0.6943322763474483,0.32210086748537925,0.29577464788732394,0.9881288275749126,213 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.9021909233176838,0.5789579680053694,0.13548606467162275,0.892018779342723,0.8906297555939263,213 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9518779342723005,0.6898709692218071,0.13548606467162275,0.9389671361502347,0.9807905334864873,213 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9197965571205008,2.6943514650786105,0.6839666644937138,0.9765258215962441,4.190030688848515,213 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9655712050078247,3.210517791199548,0.6839666644937138,0.9859154929577465,4.601967336099581,213 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9194053208137716,3.4922540819240244,0.9237972600914225,0.9812206572769953,5.388123646746446,213 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9698748043818467,4.16127740078602,0.9237972600914225,1.0,5.930060622023973,213 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9354460093896714,2.1911606588763415,0.5011619074030041,0.9624413145539906,3.4237130298632317,213 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.969092331768388,2.6109289637511237,0.5011619074030041,0.9953051643192489,3.7555769901088496,213 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9139280125195618,1.109267010415937,0.2755940893225088,0.8685446009389671,1.7291751019525257,213 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.954225352112676,1.321773168159112,0.2755940893225088,0.9342723004694836,1.8979923185233545,213 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9205790297339593,1.4158326633799614,0.33401929336661684,0.9107981220657277,2.1920950306133054,213 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9616588419405321,1.6870686746170975,0.33401929336661684,0.9389671361502347,2.408946885913222,213 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9123630672926448,1.0199912152288533,0.24510901682198524,0.9248826291079812,1.5958111429274424,213 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9561815336463223,1.2153944968957262,0.24510901682198524,0.971830985915493,1.7484330943458928,213 +Linear,Logistic,experimental,False,1,0.9,0.8450704225352113,0.29447476044239895,0.08354776001109418,0.7276995305164319,0.4596087013280646,213 +Linear,Logistic,experimental,False,1,0.95,0.9084507042253521,0.35088831940192455,0.08354776001109418,0.8075117370892019,0.5045368833572496,213 +Linear,Logistic,experimental,False,4,0.9,0.3341158059467919,0.9748301672712222,0.7902188560164397,0.056338028169014086,1.4111319523263854,213 +Linear,Logistic,experimental,False,4,0.95,0.4017996870109546,1.1615817891564408,0.7902188560164397,0.07042253521126761,1.5730924328858458,213 +Linear,Logistic,experimental,False,6,0.9,0.9080594679186228,0.9825593405176763,0.23361443805782198,0.9154929577464789,1.4168592399518378,213 +Linear,Logistic,experimental,False,6,0.95,0.956964006259781,1.1707916671328766,0.23361443805782198,0.9483568075117371,1.5807909190964573,213 +Linear,Logistic,experimental,True,1,0.9,0.8438967136150235,0.29448998280268696,0.08353477895677355,0.7183098591549296,0.45943729886831725,213 +Linear,Logistic,experimental,True,1,0.95,0.9088419405320814,0.35090645796297054,0.08353477895677355,0.812206572769953,0.5041114400484898,213 +Linear,Logistic,experimental,True,4,0.9,0.3333333333333333,0.9747975428898821,0.7903965236641811,0.056338028169014086,1.4119051235101507,213 +Linear,Logistic,experimental,True,4,0.95,0.4033646322378717,1.161542914808355,0.7903965236641811,0.07981220657276995,1.5743781294233699,213 +Linear,Logistic,experimental,True,6,0.9,0.9092331768388106,0.9825623304514758,0.2341476257220959,0.92018779342723,1.41979846440565,213 +Linear,Logistic,experimental,True,6,0.95,0.9553990610328639,1.1707952298587427,0.2341476257220959,0.9530516431924883,1.5817945290736959,213 +Linear,Logistic,observational,False,1,0.9,0.894757433489828,0.3183557829503854,0.07809759992541833,0.892018779342723,0.49488785729946616,213 +Linear,Logistic,observational,False,1,0.95,0.9483568075117371,0.3793443128488264,0.07809759992541833,0.9436619718309859,0.5435074166885125,213 +Linear,Logistic,observational,False,4,0.9,0.44405320813771515,1.2415493083413676,0.7711800746228612,0.19718309859154928,1.774858650110625,213 +Linear,Logistic,observational,False,4,0.95,0.5453834115805947,1.4793972481853468,0.7711800746228612,0.2676056338028169,1.9858757227772874,213 +Linear,Logistic,observational,False,6,0.9,0.9115805946791862,1.0304949773365348,0.24398615030125753,0.92018779342723,1.4858759045886074,213 +Linear,Logistic,observational,False,6,0.95,0.9557902973395932,1.2279105014178964,0.24398615030125753,0.9577464788732394,1.6602060040028728,213 +Linear,Logistic,observational,True,1,0.9,0.892018779342723,0.3163792430260323,0.07808090534806102,0.8873239436619719,0.4927677081751961,213 +Linear,Logistic,observational,True,1,0.95,0.9464006259780908,0.37698912026374665,0.07808090534806102,0.9530516431924883,0.5411219121582408,213 +Linear,Logistic,observational,True,4,0.9,0.4409233176838811,1.2414842562609323,0.7671739690013839,0.19718309859154928,1.7725218888802696,213 +Linear,Logistic,observational,True,4,0.95,0.5410798122065728,1.4793197338505248,0.7671739690013839,0.28169014084507044,1.9817316211484592,213 +Linear,Logistic,observational,True,6,0.9,0.9061032863849765,1.027459591935346,0.24734972591149948,0.9154929577464789,1.4817614944804902,213 +Linear,Logistic,observational,True,6,0.95,0.9546165884194053,1.2242936166276337,0.24734972591149948,0.9436619718309859,1.6552966469110102,213 diff --git a/results/did/did_multi_eventstudy.csv b/results/did/did_multi_eventstudy.csv index 2977684..2498639 100644 --- a/results/did/did_multi_eventstudy.csv +++ b/results/did/did_multi_eventstudy.csv @@ -1,49 +1,49 @@ Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,experimental,False,1,0.9,0.2695,0.6664379809971152,0.5273245725879285,0.065,0.875601005136764,1000 -LGBM,LGBM,experimental,False,1,0.95,0.35933333333333334,0.7941098340189701,0.5273245725879285,0.123,0.993123730170564,1000 -LGBM,LGBM,experimental,False,4,0.9,0.398,0.5415820487655375,0.3712684690429321,0.2,0.7377555251101078,1000 -LGBM,LGBM,experimental,False,4,0.95,0.4905,0.6453348145154958,0.3712684690429321,0.303,0.8304953131373951,1000 -LGBM,LGBM,experimental,False,6,0.9,0.8975,0.5398952913714347,0.13395236250758816,0.89,0.7349862691616095,1000 -LGBM,LGBM,experimental,False,6,0.95,0.9498333333333334,0.6433249191126899,0.13395236250758816,0.943,0.8272174489329279,1000 -LGBM,LGBM,experimental,True,1,0.9,0.27066666666666667,0.6664609685957759,0.5268935068048141,0.073,0.8756450610322313,1000 -LGBM,LGBM,experimental,True,1,0.95,0.362,0.7941372254322411,0.5268935068048141,0.131,0.9924410930584766,1000 -LGBM,LGBM,experimental,True,4,0.9,0.3908333333333333,0.5414895331777497,0.37122437324957164,0.19,0.7376044296248573,1000 -LGBM,LGBM,experimental,True,4,0.95,0.4928333333333333,0.6452245753932411,0.37122437324957164,0.294,0.8300401173811606,1000 -LGBM,LGBM,experimental,True,6,0.9,0.8971666666666667,0.5397522456658546,0.13239266622447163,0.893,0.7343809360201817,1000 -LGBM,LGBM,experimental,True,6,0.95,0.9475,0.6431544696413897,0.13239266622447163,0.946,0.8273225484496115,1000 -LGBM,LGBM,observational,False,1,0.9,0.902,2.653721378497191,0.7070490800856716,0.933,3.6804836186191254,1000 -LGBM,LGBM,observational,False,1,0.95,0.958,3.162104056941675,0.7070490800856716,0.973,4.1198480194456915,1000 -LGBM,LGBM,observational,False,4,0.9,0.901,3.561000033087922,1.0237738258563158,0.94,4.872767022094824,1000 -LGBM,LGBM,observational,False,4,0.95,0.9645,4.243193254061008,1.0237738258563158,0.99,5.476875359796976,1000 -LGBM,LGBM,observational,False,6,0.9,0.932,2.0204226296120424,0.46987471691349253,0.953,2.8163439845272875,1000 -LGBM,LGBM,observational,False,6,0.95,0.9728333333333333,2.4074820535421066,0.46987471691349253,0.982,3.1491467155171455,1000 -LGBM,LGBM,observational,True,1,0.9,0.9123333333333333,1.07183187495378,0.2603956402406786,0.938,1.4925039393303094,1000 -LGBM,LGBM,observational,True,1,0.95,0.9661666666666666,1.2771664529718216,0.2603956402406786,0.969,1.6679753377448512,1000 -LGBM,LGBM,observational,True,4,0.9,0.936,1.3868740368469075,0.30530138471391266,0.942,1.9070687685924095,1000 -LGBM,LGBM,observational,True,4,0.95,0.9701666666666666,1.6525623427973326,0.30530138471391266,0.973,2.1382550302116794,1000 -LGBM,LGBM,observational,True,6,0.9,0.9173333333333333,0.9435499321525795,0.22352810241642249,0.928,1.3193862165661938,1000 -LGBM,LGBM,observational,True,6,0.95,0.96,1.1243090900810153,0.22352810241642249,0.965,1.4751970915806725,1000 -Linear,Linear,experimental,False,1,0.9,0.8038333333333334,0.21012774921404478,0.06511858439798791,0.723,0.29993875767609723,1000 -Linear,Linear,experimental,False,1,0.95,0.8825,0.25038265646487395,0.06511858439798791,0.819,0.3333237643577245,1000 -Linear,Linear,experimental,False,4,0.9,0.18866666666666665,0.9724995300651125,0.9445552734461886,0.035,1.2537683002940827,1000 -Linear,Linear,experimental,False,4,0.95,0.2515,1.1588046636358738,0.9445552734461886,0.059,1.4271667404908353,1000 -Linear,Linear,experimental,False,6,0.9,0.8888333333333334,0.9839291437722918,0.24625318596473464,0.885,1.265266070206284,1000 -Linear,Linear,experimental,False,6,0.95,0.941,1.172423888384033,0.24625318596473464,0.936,1.4400279208463234,1000 -Linear,Linear,experimental,True,1,0.9,0.8053333333333333,0.21013098816621933,0.06512604019501264,0.732,0.300014012964527,1000 -Linear,Linear,experimental,True,1,0.95,0.8816666666666666,0.2503865159144358,0.06512604019501264,0.824,0.333092537873174,1000 -Linear,Linear,experimental,True,4,0.9,0.18933333333333333,0.972518967504493,0.9443719698591424,0.037,1.2555177892281149,1000 -Linear,Linear,experimental,True,4,0.95,0.252,1.1588278247734445,0.9443719698591424,0.06,1.4274806534380786,1000 -Linear,Linear,experimental,True,6,0.9,0.8875,0.9839063096253385,0.24613101616711022,0.885,1.2651613665652979,1000 -Linear,Linear,experimental,True,6,0.95,0.9415,1.1723966798197494,0.24613101616711022,0.941,1.4397415352658391,1000 -Linear,Linear,observational,False,1,0.9,0.8916666666666666,0.22637212149559552,0.054683601823785656,0.891,0.3228630405513039,1000 -Linear,Linear,observational,False,1,0.95,0.9456666666666667,0.2697390199136439,0.054683601823785656,0.949,0.358552269863231,1000 -Linear,Linear,observational,False,4,0.9,0.3156666666666667,1.2872024456860576,0.9190699365216015,0.185,1.6388325707815383,1000 -Linear,Linear,observational,False,4,0.95,0.4146666666666667,1.5337963165952757,0.9190699365216015,0.262,1.8700830647518032,1000 -Linear,Linear,observational,False,6,0.9,0.8868333333333334,1.0372399263148084,0.26320551491048766,0.877,1.3311336840392576,1000 -Linear,Linear,observational,False,6,0.95,0.9406666666666667,1.2359476038435253,0.26320551491048766,0.943,1.5174947513822885,1000 -Linear,Linear,observational,True,1,0.9,0.8923333333333334,0.22495023196577907,0.05470760964666116,0.891,0.3207319286670861,1000 -Linear,Linear,observational,True,1,0.95,0.9435,0.2680447340375201,0.05470760964666116,0.94,0.35645975459527846,1000 -Linear,Linear,observational,True,4,0.9,0.3105,1.2870401885570502,0.9190767720949804,0.177,1.6393372900395775,1000 -Linear,Linear,observational,True,4,0.95,0.41433333333333333,1.533602975301024,0.9190767720949804,0.257,1.8708048686939287,1000 -Linear,Linear,observational,True,6,0.9,0.8815,1.028838214628254,0.26320130535902825,0.884,1.3218761748802963,1000 -Linear,Linear,observational,True,6,0.95,0.9405,1.225936346887698,0.26320130535902825,0.935,1.5055011269416756,1000 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.2793427230046948,0.6538940149561205,0.5211711102717896,0.06103286384976526,0.86055821069107,213 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.3528951486697966,0.7791627765660771,0.5211711102717896,0.1267605633802817,0.9765429620402541,213 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.39358372456964,0.541925194741234,0.3658723164598346,0.14084507042253522,0.7380373644216336,213 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.4694835680751174,0.6457436981649495,0.3658723164598346,0.30985915492957744,0.8302776266426216,213 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.9014084507042254,0.5382900596052185,0.12834769643838123,0.9014084507042254,0.7329552647921785,213 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9499217527386542,0.6414121674872129,0.12834769643838123,0.9530516431924883,0.8240621744454719,213 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.2902973395931142,0.653663670745508,0.5177678460962447,0.07511737089201878,0.8606157129718126,213 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.36932707355242567,0.7788883045100532,0.5177678460962447,0.14084507042253522,0.978192241517096,213 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.3763693270735525,0.5420271299376794,0.36469586751312316,0.18779342723004694,0.7374141258484136,213 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.4647887323943662,0.645865161443211,0.36469586751312316,0.27230046948356806,0.8307165715799049,213 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.9194053208137716,0.5383523983929166,0.12524713080884226,0.9248826291079812,0.7331206282884405,213 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9483568075117371,0.6414864487343258,0.12524713080884226,0.9389671361502347,0.8244543573667497,213 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9131455399061033,2.635762776399446,0.6863198917243725,0.9389671361502347,3.6611895504458305,213 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9608763693270735,3.1407050626801003,0.6863198917243725,0.9812206572769953,4.095116662293061,213 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9107981220657277,3.5476693552380327,0.9618718220501209,0.9577464788732394,4.871536434987288,213 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9733959311424101,4.227308771668667,0.9618718220501209,0.9765258215962441,5.471362486927891,213 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.94679186228482,2.048525808104845,0.4424199042553949,0.9530516431924883,2.8547596856961075,213 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9780907668231612,2.4409690561510136,0.4424199042553949,0.9859154929577465,3.191610282696814,213 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9178403755868545,1.0506825752853373,0.25838130000929665,0.9248826291079812,1.4648011676731838,213 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9624413145539906,1.2519655080553924,0.25838130000929665,0.9577464788732394,1.6360312921813787,213 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9225352112676056,1.3999042833283855,0.32349351954396005,0.9248826291079812,1.9287116595218243,213 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9679186228482003,1.6680888391340973,0.32349351954396005,0.9624413145539906,2.1548062661512533,213 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9194053208137716,0.941095371039425,0.21448858413067756,0.9530516431924883,1.3188087131097894,213 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9710485133020345,1.1213843001174537,0.21448858413067756,0.9812206572769953,1.4720050449124056,213 +Linear,Logistic,experimental,False,1,0.9,0.8176838810641627,0.2097616817290772,0.06354647829956948,0.7511737089201878,0.29957865791433536,213 +Linear,Logistic,experimental,False,1,0.95,0.888888888888889,0.24994646015251432,0.06354647829956948,0.8356807511737089,0.3324902033558004,213 +Linear,Logistic,experimental,False,4,0.9,0.20735524256651017,0.9731315969907915,0.9247823933778377,0.051643192488262914,1.2553334876224438,213 +Linear,Logistic,experimental,False,4,0.95,0.26682316118935834,1.1595578178313908,0.9247823933778377,0.06572769953051644,1.427245013596636,213 +Linear,Logistic,experimental,False,6,0.9,0.9162754303599373,0.982583831624362,0.2340222441493123,0.9107981220657277,1.2642250097051084,213 +Linear,Logistic,experimental,False,6,0.95,0.9538341158059468,1.1708208500864592,0.2340222441493123,0.9436619718309859,1.440074754119655,213 +Linear,Logistic,experimental,True,1,0.9,0.8184663536776213,0.209784288558112,0.06346289348873158,0.7511737089201878,0.2994094620604152,213 +Linear,Logistic,experimental,True,1,0.95,0.8896713615023474,0.24997339785079145,0.06346289348873158,0.8497652582159625,0.33304580863290817,213 +Linear,Logistic,experimental,True,4,0.9,0.21048513302034427,0.9730155853528542,0.9248338055294343,0.051643192488262914,1.2540423515599672,213 +Linear,Logistic,experimental,True,4,0.95,0.2699530516431925,1.1594195814385477,0.9248338055294343,0.07042253521126761,1.4266453422038732,213 +Linear,Logistic,experimental,True,6,0.9,0.9139280125195618,0.9825202138375453,0.23446937547502114,0.9061032863849765,1.2635545418793521,213 +Linear,Logistic,experimental,True,6,0.95,0.9522691705790298,1.1707450448178967,0.23446937547502114,0.9389671361502347,1.4364612584863943,213 +Linear,Logistic,observational,False,1,0.9,0.9100156494522692,0.2256652512730398,0.05275781831070359,0.9248826291079812,0.3210767224862521,213 +Linear,Logistic,observational,False,1,0.95,0.960093896713615,0.26889673209225234,0.05275781831070359,0.9483568075117371,0.357572619911957,213 +Linear,Logistic,observational,False,4,0.9,0.3325508607198748,1.2935393301274787,0.9012986658762238,0.20187793427230047,1.6449223740918557,213 +Linear,Logistic,observational,False,4,0.95,0.4460093896713615,1.5413471801346634,0.9012986658762238,0.2863849765258216,1.8802622354415948,213 +Linear,Logistic,observational,False,6,0.9,0.9123630672926448,1.0362716142235429,0.2466341904936733,0.9248826291079812,1.328413177022464,213 +Linear,Logistic,observational,False,6,0.95,0.960093896713615,1.2347937888209737,0.2466341904936733,0.9483568075117371,1.5124625048982703,213 +Linear,Logistic,observational,True,1,0.9,0.906885758998435,0.22449816162535258,0.05285134622236858,0.9107981220657277,0.32006341717518294,213 +Linear,Logistic,observational,True,1,0.95,0.9561815336463223,0.267506059002127,0.05285134622236858,0.9483568075117371,0.355619479598059,213 +Linear,Logistic,observational,True,4,0.9,0.3380281690140845,1.2948063187256469,0.8972215616534344,0.1784037558685446,1.6460425935700733,213 +Linear,Logistic,observational,True,4,0.95,0.430359937402191,1.5428568901663307,0.8972215616534344,0.3051643192488263,1.8803448054869478,213 +Linear,Logistic,observational,True,6,0.9,0.9123630672926448,1.0324801100143304,0.2506321361153765,0.9014084507042254,1.3236751031745002,213 +Linear,Logistic,observational,True,6,0.95,0.9499217527386542,1.2302759329002244,0.2506321361153765,0.9530516431924883,1.510018818061999,213 diff --git a/results/did/did_multi_group.csv b/results/did/did_multi_group.csv index d6d461b..de75e59 100644 --- a/results/did/did_multi_group.csv +++ b/results/did/did_multi_group.csv @@ -1,49 +1,49 @@ Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,experimental,False,1,0.9,0.3723333333333333,0.712829716785441,0.5120534147173762,0.073,0.8873091560105003,1000 -LGBM,LGBM,experimental,False,1,0.95,0.4606666666666666,0.849388996757564,0.5120534147173762,0.122,1.0124899542984358,1000 -LGBM,LGBM,experimental,False,4,0.9,0.504,0.608382718738708,0.36202329008148987,0.219,0.7745653409497698,1000 -LGBM,LGBM,experimental,False,4,0.95,0.589,0.7249327222838707,0.36202329008148987,0.31,0.8776439618079263,1000 -LGBM,LGBM,experimental,False,6,0.9,0.8986666666666666,0.6042439878313273,0.14850794162785066,0.899,0.7675396748306148,1000 -LGBM,LGBM,experimental,False,6,0.95,0.9503333333333334,0.7200011202329313,0.14850794162785066,0.945,0.8700300494781059,1000 -LGBM,LGBM,experimental,True,1,0.9,0.3866666666666666,0.7129187295177013,0.5108468478621639,0.076,0.8873896060492763,1000 -LGBM,LGBM,experimental,True,1,0.95,0.4633333333333333,0.8494950619700162,0.5108468478621639,0.135,1.0122432445151552,1000 -LGBM,LGBM,experimental,True,4,0.9,0.5016666666666666,0.608263026948519,0.3616198619719443,0.223,0.7738821674571664,1000 -LGBM,LGBM,experimental,True,4,0.95,0.5966666666666667,0.7247901007191481,0.3616198619719443,0.308,0.877557935434779,1000 -LGBM,LGBM,experimental,True,6,0.9,0.8976666666666666,0.6041013460350053,0.14653383224031008,0.898,0.7676921917774191,1000 -LGBM,LGBM,experimental,True,6,0.95,0.947,0.7198311520491973,0.14653383224031008,0.954,0.8702372370829647,1000 -LGBM,LGBM,observational,False,1,0.9,0.9093333333333333,2.665571501346606,0.6783738614604253,0.937,3.372964798983418,1000 -LGBM,LGBM,observational,False,1,0.95,0.9673333333333334,3.1762243492380025,0.6783738614604253,0.98,3.825961439092819,1000 -LGBM,LGBM,observational,False,4,0.9,0.909,3.534531876327934,0.9946875737404597,0.935,4.445144884485254,1000 -LGBM,LGBM,observational,False,4,0.95,0.967,4.211654500012185,0.9946875737404597,0.976,5.055292173043325,1000 -LGBM,LGBM,observational,False,6,0.9,0.9416666666666667,2.122820536068101,0.4787787714125554,0.957,2.6963338440693168,1000 -LGBM,LGBM,observational,False,6,0.95,0.9766666666666667,2.5294966847881346,0.4787787714125554,0.986,3.0585559742251704,1000 -LGBM,LGBM,observational,True,1,0.9,0.921,1.1147256542022042,0.2672559046096273,0.934,1.4159126956485517,1000 -LGBM,LGBM,observational,True,1,0.95,0.966,1.3282775434118486,0.2672559046096273,0.968,1.6037050855151684,1000 -LGBM,LGBM,observational,True,4,0.9,0.94,1.422767817772445,0.30810115099970276,0.936,1.7975736984737825,1000 -LGBM,LGBM,observational,True,4,0.95,0.9706666666666667,1.6953324207728482,0.30810115099970276,0.972,2.041855425299719,1000 -LGBM,LGBM,observational,True,6,0.9,0.91,1.006741237739991,0.237584782868055,0.927,1.2840461885633785,1000 -LGBM,LGBM,observational,True,6,0.95,0.9603333333333334,1.1996061749145979,0.237584782868055,0.969,1.4521486558429435,1000 -Linear,Linear,experimental,False,1,0.9,0.809,0.2639472488371848,0.07836725122654463,0.751,0.33918155040956693,1000 -Linear,Linear,experimental,False,1,0.95,0.8853333333333334,0.3145125457139394,0.07836725122654463,0.833,0.3826104197586172,1000 -Linear,Linear,experimental,False,4,0.9,0.29933333333333334,1.0775733624029018,0.9144760793156517,0.031,1.3581931890994936,1000 -Linear,Linear,experimental,False,4,0.95,0.376,1.28400785723636,0.9144760793156517,0.066,1.544378342112665,1000 -Linear,Linear,experimental,False,6,0.9,0.8916666666666666,1.0855871936277797,0.2674422239505616,0.886,1.3663373525384621,1000 -Linear,Linear,experimental,False,6,0.95,0.9413333333333334,1.293556926114941,0.2674422239505616,0.947,1.554433869195187,1000 -Linear,Linear,experimental,True,1,0.9,0.809,0.26395101798234016,0.07841196950190694,0.751,0.3392834245796571,1000 -Linear,Linear,experimental,True,1,0.95,0.8856666666666666,0.3145170369274041,0.07841196950190694,0.832,0.38257509241292276,1000 -Linear,Linear,experimental,True,4,0.9,0.29733333333333334,1.0775450811401135,0.9139676817765434,0.033,1.3594859465438789,1000 -Linear,Linear,experimental,True,4,0.95,0.37666666666666665,1.283974158033225,0.9139676817765434,0.065,1.545038224259449,1000 -Linear,Linear,experimental,True,6,0.9,0.8946666666666666,1.0856219017160036,0.2672586672222071,0.896,1.366450981317095,1000 -Linear,Linear,experimental,True,6,0.95,0.942,1.2935982833529203,0.2672586672222071,0.952,1.5542102827957853,1000 -Linear,Linear,observational,False,1,0.9,0.8963333333333334,0.28417672167828534,0.0704315330527404,0.888,0.36504143473823225,1000 -Linear,Linear,observational,False,1,0.95,0.945,0.33861744936319155,0.0704315330527404,0.941,0.41189946472127525,1000 -Linear,Linear,observational,False,4,0.9,0.408,1.3757050815726335,0.8986741240953303,0.193,1.723360009232728,1000 -Linear,Linear,observational,False,4,0.95,0.5103333333333333,1.639253711728994,0.8986741240953303,0.289,1.9608953756229532,1000 -Linear,Linear,observational,False,6,0.9,0.8906666666666666,1.1373073172966048,0.284219716886748,0.88,1.4298799083888758,1000 -Linear,Linear,observational,False,6,0.95,0.9396666666666667,1.35518525462143,0.284219716886748,0.944,1.6242483562467662,1000 -Linear,Linear,observational,True,1,0.9,0.8976666666666666,0.28247229568225285,0.07026934090791014,0.889,0.3628745212929247,1000 -Linear,Linear,observational,True,1,0.95,0.9476666666666667,0.33658650052263794,0.07026934090791014,0.945,0.40971246171295345,1000 -Linear,Linear,observational,True,4,0.9,0.4056666666666666,1.3749682398133307,0.8988135221197031,0.198,1.7216124241871145,1000 -Linear,Linear,observational,True,4,0.95,0.505,1.638375710618819,0.8988135221197031,0.282,1.959504408369747,1000 -Linear,Linear,observational,True,6,0.9,0.8893333333333334,1.1272233554666016,0.2838441687177051,0.879,1.416566966857148,1000 -Linear,Linear,observational,True,6,0.95,0.943,1.3431694729832098,0.2838441687177051,0.945,1.6114267967021605,1000 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.38341158059467917,0.7010247560032261,0.5045579001393814,0.06572769953051644,0.8747944725382377,213 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.44913928012519555,0.8353225183834784,0.5045579001393814,0.13145539906103287,0.9956978967225759,213 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.514866979655712,0.6082806742773996,0.35754266882547,0.18309859154929578,0.7739715228608116,213 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.5884194053208137,0.7248111288084945,0.35754266882547,0.29107981220657275,0.878325999451341,213 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.9045383411580594,0.6032942628400023,0.1416087698948157,0.9061032863849765,0.7672596153857486,213 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9577464788732394,0.7188694531060122,0.1416087698948157,0.9483568075117371,0.8700719219107395,213 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.38028169014084506,0.7006624811428777,0.5027623497763666,0.08450704225352113,0.8749076136062413,213 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.45383411580594674,0.8348908412620905,0.5027623497763666,0.14084507042253522,0.9960800157981575,213 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.49765258215962443,0.6078684171461785,0.3564700998360187,0.1784037558685446,0.7724813665598308,213 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.5915492957746479,0.7243198941379295,0.3564700998360187,0.28169014084507044,0.8768342044058519,213 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.9092331768388106,0.6035874382384104,0.13992167424040736,0.9295774647887324,0.7668293165413835,213 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9577464788732394,0.7192187931400541,0.13992167424040736,0.9483568075117371,0.8695905007790928,213 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9233176838810642,2.6401625405977405,0.6680316774131236,0.9530516431924883,3.348805972231518,213 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9687010954616588,3.1459477050817273,0.6680316774131236,0.9812206572769953,3.791143326987578,213 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9123630672926448,3.4982482043306202,0.9449299713795152,0.9248826291079812,4.416321409189137,213 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9577464788732394,4.1684198381697195,0.9449299713795152,0.9812206572769953,5.007980964364257,213 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9389671361502347,2.1556775041672305,0.46296654578958996,0.9483568075117371,2.739619518142511,213 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9733959311424101,2.568648177091331,0.46296654578958996,0.9906103286384976,3.1020251380422854,213 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9264475743348983,1.0864056229119294,0.2572919037765861,0.9389671361502347,1.3803957820627104,213 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9624413145539906,1.2945321447572218,0.2572919037765861,0.9624413145539906,1.5630008344403068,213 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9405320813771518,1.4335989322710556,0.318773509562249,0.9577464788732394,1.8127247140484581,213 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9765258215962441,1.7082384897274767,0.318773509562249,0.9859154929577465,2.0590624139128217,213 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.917057902973396,1.015686570779001,0.23606871293652523,0.9483568075117371,1.2952585886030479,213 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9640062597809077,1.2102651966651654,0.23606871293652523,0.9812206572769953,1.4663039002495655,213 +Linear,Logistic,experimental,False,1,0.9,0.8169014084507042,0.263859732601683,0.08071876563864448,0.7746478873239436,0.33907438784389654,213 +Linear,Logistic,experimental,False,1,0.95,0.895148669796557,0.31440826368735936,0.08071876563864448,0.8544600938967136,0.38284950361525444,213 +Linear,Logistic,experimental,False,4,0.9,0.3302034428794992,1.077782919659112,0.8937614998777182,0.051643192488262914,1.3561544020534126,213 +Linear,Logistic,experimental,False,4,0.95,0.39749608763693267,1.2842575601084827,0.8937614998777182,0.07511737089201878,1.5423676656010097,213 +Linear,Logistic,experimental,False,6,0.9,0.9092331768388106,1.084908190453934,0.25576007865798384,0.9295774647887324,1.365127434841,213 +Linear,Logistic,experimental,False,6,0.95,0.9593114241001566,1.2927478439301676,0.25576007865798384,0.9577464788732394,1.55476043284721,213 +Linear,Logistic,experimental,True,1,0.9,0.8106416275430359,0.26388645094216934,0.08082008766978688,0.7793427230046949,0.33860699865003,213 +Linear,Logistic,experimental,True,1,0.95,0.8982785602503913,0.31444010055370525,0.08082008766978688,0.8544600938967136,0.38363187019926315,213 +Linear,Logistic,experimental,True,4,0.9,0.32707355242566505,1.0776870925219295,0.8937933391766094,0.051643192488262914,1.3587107289972284,213 +Linear,Logistic,experimental,True,4,0.95,0.40219092331768386,1.284143375031743,0.8937933391766094,0.07981220657276995,1.5423186997471712,213 +Linear,Logistic,experimental,True,6,0.9,0.9123630672926448,1.0847636730976646,0.25673459846880975,0.9295774647887324,1.3684316527340743,213 +Linear,Logistic,experimental,True,6,0.95,0.9530516431924883,1.2925756408789129,0.25673459846880975,0.9530516431924883,1.552030854337275,213 +Linear,Logistic,observational,False,1,0.9,0.895148669796557,0.2840761342590122,0.06969384593021301,0.9154929577464789,0.36506933928947094,213 +Linear,Logistic,observational,False,1,0.95,0.9577464788732394,0.3384975920604852,0.06969384593021301,0.9483568075117371,0.41184024131390085,213 +Linear,Logistic,observational,False,4,0.9,0.42879499217527384,1.3815569998881347,0.8834464565663525,0.19718309859154928,1.7278992279793388,213 +Linear,Logistic,observational,False,4,0.95,0.5226917057902973,1.646226702486907,0.8834464565663525,0.2863849765258216,1.9664187739003602,213 +Linear,Logistic,observational,False,6,0.9,0.9139280125195618,1.1353911192689263,0.2657678059270839,0.92018779342723,1.4244455772241778,213 +Linear,Logistic,observational,False,6,0.95,0.9593114241001566,1.3529019638410478,0.2657678059270839,0.9624413145539906,1.6176444941500485,213 +Linear,Logistic,observational,True,1,0.9,0.8826291079812206,0.2823311571824021,0.06993758010197051,0.9295774647887324,0.3628548286921628,213 +Linear,Logistic,observational,True,1,0.95,0.9561815336463223,0.33641832362713375,0.06993758010197051,0.9577464788732394,0.40938919268144514,213 +Linear,Logistic,observational,True,4,0.9,0.4194053208137715,1.3815154955504438,0.876582736122305,0.2112676056338028,1.7315155396141317,213 +Linear,Logistic,observational,True,4,0.95,0.5273865414710485,1.6461772470181997,0.876582736122305,0.3145539906103286,1.9679400718614783,213 +Linear,Logistic,observational,True,6,0.9,0.9076682316118936,1.1396624657364007,0.2717053464179361,0.9154929577464789,1.430838330810308,213 +Linear,Logistic,observational,True,6,0.95,0.9608763693270735,1.3579915870783799,0.2717053464179361,0.9577464788732394,1.6241526383361697,213 diff --git a/results/did/did_multi_metadata.csv b/results/did/did_multi_metadata.csv index 191c4eb..67772b0 100644 --- a/results/did/did_multi_metadata.csv +++ b/results/did/did_multi_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.10.dev0,DIDMultiCoverageSimulation,2025-05-23 09:01,152.65548847913743,3.12.9,scripts/did/did_pa_multi_config.yml +0.11.dev0,DIDMultiCoverageSimulation,2025-06-02 17:51,339.9238995909691,3.12.3,scripts/did/did_pa_multi_config.yml diff --git a/results/did/did_multi_time.csv b/results/did/did_multi_time.csv index 333f774..8a5c303 100644 --- a/results/did/did_multi_time.csv +++ b/results/did/did_multi_time.csv @@ -1,49 +1,49 @@ Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,experimental,False,1,0.9,0.105,0.6770474680143006,0.5882187359928296,0.06,0.8037202643084942,1000 -LGBM,LGBM,experimental,False,1,0.95,0.17066666666666666,0.8067518175410349,0.5882187359928296,0.118,0.926841610902776,1000 -LGBM,LGBM,experimental,False,4,0.9,0.236,0.5444213002720629,0.41426795329176225,0.178,0.6607984097007097,1000 -LGBM,LGBM,experimental,False,4,0.95,0.3276666666666666,0.6487179913554645,0.41426795329176225,0.267,0.7587562686524103,1000 -LGBM,LGBM,experimental,False,6,0.9,0.8943333333333334,0.53982673853458,0.1342938994875208,0.908,0.6564456680014384,1000 -LGBM,LGBM,experimental,False,6,0.95,0.95,0.6432432333693072,0.1342938994875208,0.951,0.7533485914407448,1000 -LGBM,LGBM,experimental,True,1,0.9,0.104,0.6773164375641183,0.5871963271333144,0.063,0.8038330391648153,1000 -LGBM,LGBM,experimental,True,1,0.95,0.17566666666666667,0.8070723145274795,0.5871963271333144,0.136,0.9270642436123595,1000 -LGBM,LGBM,experimental,True,4,0.9,0.2313333333333333,0.5443770179339803,0.414059062792942,0.171,0.6603179410869597,1000 -LGBM,LGBM,experimental,True,4,0.95,0.327,0.6486652256951217,0.414059062792942,0.258,0.7584758454668876,1000 -LGBM,LGBM,experimental,True,6,0.9,0.9013333333333333,0.5395668449941098,0.13154818699839163,0.895,0.6563984364887405,1000 -LGBM,LGBM,experimental,True,6,0.95,0.9493333333333334,0.6429335511150388,0.13154818699839163,0.942,0.7529872439559875,1000 -LGBM,LGBM,observational,False,1,0.9,0.8973333333333333,2.8900944626182756,0.7577316621702538,0.915,3.57007703127756,1000 -LGBM,LGBM,observational,False,1,0.95,0.958,3.4437599588413588,0.7577316621702538,0.968,4.083723276817505,1000 -LGBM,LGBM,observational,False,4,0.9,0.8883333333333334,3.944278944663819,1.1508934356199159,0.921,4.817695497037129,1000 -LGBM,LGBM,observational,False,4,0.95,0.959,4.699898246173131,1.1508934356199159,0.981,5.525753606365474,1000 -LGBM,LGBM,observational,False,6,0.9,0.9346666666666666,2.007004968725184,0.4621685573032238,0.943,2.5048024891920218,1000 -LGBM,LGBM,observational,False,6,0.95,0.973,2.391493924468424,0.4621685573032238,0.976,2.8553234091559982,1000 -LGBM,LGBM,observational,True,1,0.9,0.9203333333333333,1.1214028688438904,0.2681238910047797,0.932,1.3877940365894852,1000 -LGBM,LGBM,observational,True,1,0.95,0.966,1.336233935397319,0.2681238910047797,0.965,1.5830862634102072,1000 -LGBM,LGBM,observational,True,4,0.9,0.946,1.4738907100586922,0.3155685506978381,0.947,1.8023292945303246,1000 -LGBM,LGBM,observational,True,4,0.95,0.975,1.756249104193653,0.3155685506978381,0.978,2.0652293790201095,1000 -LGBM,LGBM,observational,True,6,0.9,0.9036666666666666,0.9243181245460811,0.22145829614971912,0.904,1.1513913386950982,1000 -LGBM,LGBM,observational,True,6,0.95,0.9506666666666667,1.101392977881901,0.22145829614971912,0.964,1.312498872874679,1000 -Linear,Linear,experimental,False,1,0.9,0.785,0.24421028738791545,0.07700598814984021,0.726,0.31283981302906455,1000 -Linear,Linear,experimental,False,1,0.95,0.8656666666666666,0.29099450558503215,0.07700598814984021,0.822,0.3536648083533155,1000 -Linear,Linear,experimental,False,4,0.9,0.029,0.9657408497851493,1.0738907262320931,0.024,1.1066254403295672,1000 -Linear,Linear,experimental,False,4,0.95,0.05633333333333333,1.1507511993551036,1.0738907262320931,0.042,1.2857511031369329,1000 -Linear,Linear,experimental,False,6,0.9,0.891,0.9640961231527081,0.2403665468723713,0.887,1.1081712902827776,1000 -Linear,Linear,experimental,False,6,0.95,0.944,1.1487913866938562,0.2403665468723713,0.945,1.2857969535196567,1000 -Linear,Linear,experimental,True,1,0.9,0.7856666666666666,0.24421161220168236,0.07699427764290885,0.725,0.312745321017087,1000 -Linear,Linear,experimental,True,1,0.95,0.866,0.2909960841980021,0.07699427764290885,0.82,0.35353627547158184,1000 -Linear,Linear,experimental,True,4,0.9,0.029333333333333333,0.9658114690308013,1.073411274232068,0.025,1.1072936001699796,1000 -Linear,Linear,experimental,True,4,0.95,0.056,1.1508353473764386,1.073411274232068,0.047,1.2871025490442616,1000 -Linear,Linear,experimental,True,6,0.9,0.8933333333333334,0.964156052938097,0.23997981769673798,0.891,1.1072565217168384,1000 -Linear,Linear,experimental,True,6,0.95,0.944,1.1488627974376686,0.23997981769673798,0.947,1.287096789672995,1000 -Linear,Linear,observational,False,1,0.9,0.889,0.2746570982471215,0.06835594688685424,0.877,0.35201642493170093,1000 -Linear,Linear,observational,False,1,0.95,0.9386666666666666,0.32727411840307125,0.06835594688685424,0.932,0.39803367647348065,1000 -Linear,Linear,observational,False,4,0.9,0.16033333333333336,1.3480635210832488,1.0552310812916714,0.131,1.518276214952089,1000 -Linear,Linear,observational,False,4,0.95,0.24266666666666667,1.6063167608976374,1.0552310812916714,0.211,1.7707505491415911,1000 -Linear,Linear,observational,False,6,0.9,0.8903333333333334,1.0186244057373655,0.2564255464027685,0.88,1.171669804736589,1000 -Linear,Linear,observational,False,6,0.95,0.9423333333333334,1.2137658429333622,0.2564255464027685,0.942,1.359505036566505,1000 -Linear,Linear,observational,True,1,0.9,0.8853333333333334,0.27250071547513943,0.06816094516072957,0.876,0.349158428206522,1000 -Linear,Linear,observational,True,1,0.95,0.9393333333333334,0.3247046298475451,0.06816094516072957,0.927,0.39461799076066073,1000 -Linear,Linear,observational,True,4,0.9,0.15733333333333335,1.348359701155594,1.0540447813311582,0.131,1.518475986344912,1000 -Linear,Linear,observational,True,4,0.95,0.24566666666666664,1.6066696812215027,1.0540447813311582,0.213,1.7697898872953792,1000 -Linear,Linear,observational,True,6,0.9,0.886,1.007087091732094,0.2561924661932343,0.886,1.1588097256348897,1000 -Linear,Linear,observational,True,6,0.95,0.9426666666666667,1.2000182853646246,0.2561924661932343,0.938,1.3446481778510444,1000 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.10485133020344288,0.6641311784095322,0.5823494355664303,0.08450704225352113,0.7881878804378276,213 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.17683881064162754,0.7913611091980961,0.5823494355664303,0.12206572769953052,0.9120924244627483,213 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.2222222222222222,0.5457603564040948,0.40420421511646165,0.15023474178403756,0.6619778117977112,213 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.34585289514866974,0.6503135751503121,0.40420421511646165,0.23943661971830985,0.7592098705244504,213 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.9107981220657277,0.5381868903349744,0.12772384273949355,0.9014084507042254,0.6551761588151264,213 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9530516431924883,0.6412892337193222,0.12772384273949355,0.9389671361502347,0.7509412548475333,213 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.10328638497652583,0.6638809210012889,0.5795783394754952,0.09389671361502347,0.7886190860615776,213 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.17527386541471046,0.7910629091035805,0.5795783394754952,0.1267605633802817,0.9100402277563315,213 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.2363067292644757,0.5454472863046809,0.40351084017097694,0.15492957746478872,0.6618382122506522,213 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.34115805946791866,0.6499405291178667,0.40351084017097694,0.2347417840375587,0.760586865832683,213 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.9045383411580594,0.5384698205489944,0.12841236979508242,0.9248826291079812,0.6548573031006936,213 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9577464788732394,0.6416263658631989,0.12841236979508242,0.9530516431924883,0.7496096842588228,213 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.917057902973396,2.826267846555917,0.6997630370632157,0.9342723004694836,3.505164210797084,213 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9624413145539906,3.367705854884923,0.6997630370632157,0.9765258215962441,4.010090570743562,213 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9061032863849765,3.9292910307842495,1.1032615377470605,0.9248826291079812,4.801480559342949,213 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9640062597809077,4.682039045253358,1.1032615377470605,0.9765258215962441,5.512445467612302,213 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9405320813771518,2.020686142877495,0.4287262814527601,0.9577464788732394,2.5196056115761056,213 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9765258215962441,2.4077960489647223,0.4287262814527601,0.9812206572769953,2.8786465586804595,213 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9420970266040688,1.0767197139991078,0.25231124461698884,0.9295774647887324,1.337727334792679,213 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9671361502347418,1.2829906724246045,0.25231124461698884,0.9671361502347418,1.521525908754415,213 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9389671361502347,1.4992929674517166,0.3404636513294964,0.9530516431924883,1.8343377037938378,213 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9780907668231612,1.7865177608087823,0.3404636513294964,0.9859154929577465,2.10956901976702,213 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9139280125195618,0.9226791828166745,0.22256304127174542,0.9248826291079812,1.1499561650632644,213 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9624413145539906,1.0994400583577786,0.22256304127174542,0.9765258215962441,1.3126410298530595,213 +Linear,Logistic,experimental,False,1,0.9,0.7981220657276995,0.24406856913925595,0.07455391600852629,0.7276995305164319,0.31257176314972324,213 +Linear,Logistic,experimental,False,1,0.95,0.863849765258216,0.29082563787621385,0.07455391600852629,0.8169014084507042,0.3530474954026036,213 +Linear,Logistic,experimental,False,4,0.9,0.046948356807511735,0.9683919238870853,1.0545074196771533,0.04225352112676056,1.1061614805926192,213 +Linear,Logistic,experimental,False,4,0.95,0.06572769953051644,1.153910148987462,1.0545074196771533,0.06103286384976526,1.2911717925367656,213 +Linear,Logistic,experimental,False,6,0.9,0.8982785602503913,0.9628742659936109,0.23137852998349606,0.892018779342723,1.1054779631893865,213 +Linear,Logistic,experimental,False,6,0.95,0.9452269170579031,1.147335454088764,0.23137852998349606,0.9624413145539906,1.289087298578374,213 +Linear,Logistic,experimental,True,1,0.9,0.8059467918622848,0.24408667991439068,0.07429939700420787,0.7370892018779343,0.3135963329097478,213 +Linear,Logistic,experimental,True,1,0.95,0.86697965571205,0.29084721819583287,0.07429939700420787,0.8262910798122066,0.35308221324877676,213 +Linear,Logistic,experimental,True,4,0.9,0.046948356807511735,0.9682481796882342,1.054760155003612,0.03286384976525822,1.1081060346576734,213 +Linear,Logistic,experimental,True,4,0.95,0.06572769953051644,1.1537388672100939,1.054760155003612,0.056338028169014086,1.2880440884321052,213 +Linear,Logistic,experimental,True,6,0.9,0.892018779342723,0.9628043535115562,0.23252849991729144,0.8873239436619719,1.1104069795500415,213 +Linear,Logistic,experimental,True,6,0.95,0.9499217527386542,1.1472521482281988,0.23252849991729144,0.9577464788732394,1.287530785006439,213 +Linear,Logistic,observational,False,1,0.9,0.9014084507042254,0.27469908871557547,0.06651861748549698,0.9014084507042254,0.3520501442016315,213 +Linear,Logistic,observational,False,1,0.95,0.9546165884194053,0.3273241531323111,0.06651861748549698,0.9436619718309859,0.39797250176020077,213 +Linear,Logistic,observational,False,4,0.9,0.18935837245696402,1.353982900128844,1.038997154582461,0.18309859154929578,1.5217737073595556,213 +Linear,Logistic,observational,False,4,0.95,0.27543035993740217,1.6133701360734638,1.038997154582461,0.26291079812206575,1.7800142309978275,213 +Linear,Logistic,observational,False,6,0.9,0.9045383411580594,1.0143473501307676,0.24543300102476093,0.9107981220657277,1.1667504579830554,213 +Linear,Logistic,observational,False,6,0.95,0.9593114241001566,1.208669416837173,0.24543300102476093,0.9577464788732394,1.3535074358284087,213 +Linear,Logistic,observational,True,1,0.9,0.8935837245696401,0.2725905663392203,0.06651898118558668,0.892018779342723,0.3498352468431658,213 +Linear,Logistic,observational,True,1,0.95,0.9530516431924883,0.32481169375566,0.06651898118558668,0.9436619718309859,0.3938902840125929,213 +Linear,Logistic,observational,True,4,0.9,0.19092331768388104,1.3586994912357533,1.029554721826275,0.1784037558685446,1.5283855406123952,213 +Linear,Logistic,observational,True,4,0.95,0.2863849765258216,1.6189903010218047,1.029554721826275,0.26291079812206575,1.7818990446425318,213 +Linear,Logistic,observational,True,6,0.9,0.895148669796557,1.0131038455549137,0.24852464288526532,0.8967136150234741,1.1643867720039867,213 +Linear,Logistic,observational,True,6,0.95,0.9530516431924883,1.2071876897440446,0.24852464288526532,0.9530516431924883,1.3557343370847632,213 diff --git a/results/did/did_pa_multi_config.yml b/results/did/did_pa_multi_config.yml index fa87158..ed4e23a 100644 --- a/results/did/did_pa_multi_config.yml +++ b/results/did/did_pa_multi_config.yml @@ -1,7 +1,8 @@ -confidence_parameters: - level: - - 0.95 - - 0.9 +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 dgp_parameters: DGP: - 1 @@ -9,115 +10,34 @@ dgp_parameters: - 6 n_obs: - 2000 -dml_parameters: - in_sample_normalization: - - true - - false - learners: - - ml_g: !!python/tuple - - Linear - - !!python/object:sklearn.linear_model._base.LinearRegression - _sklearn_version: 1.5.2 - copy_X: true - fit_intercept: true - n_jobs: null - positive: false - ml_m: !!python/tuple - - Linear - - !!python/object:sklearn.linear_model._logistic.LogisticRegression - C: 1.0 - _sklearn_version: 1.5.2 - class_weight: null - dual: false - fit_intercept: true - intercept_scaling: 1 - l1_ratio: null - max_iter: 100 - multi_class: deprecated - n_jobs: null - penalty: l2 - random_state: null - solver: lbfgs - tol: 0.0001 - verbose: 0 - warm_start: false - - ml_g: !!python/tuple - - LGBM - - !!python/object:lightgbm.sklearn.LGBMRegressor - _Booster: null - _best_iteration: -1 - _best_score: {} - _class_map: null - _class_weight: null - _classes: null - _evals_result: {} - _n_classes: -1 - _n_features: -1 - _n_features_in: -1 - _objective: null - _other_params: - verbose: -1 - boosting_type: gbdt - class_weight: null - colsample_bytree: 1.0 - importance_type: split - learning_rate: 0.02 - max_depth: -1 - min_child_samples: 20 - min_child_weight: 0.001 - min_split_gain: 0.0 +learner_definitions: + linear: &id001 + name: Linear + logistic: &id002 + name: Logistic + lgbmr: &id003 + name: LGBM Regr. + params: n_estimators: 500 - n_jobs: 1 - num_leaves: 31 - objective: null - random_state: null - reg_alpha: 0.0 - reg_lambda: 0.0 - subsample: 1.0 - subsample_for_bin: 200000 - subsample_freq: 0 - verbose: -1 - ml_m: !!python/tuple - - LGBM - - !!python/object:lightgbm.sklearn.LGBMClassifier - _Booster: null - _best_iteration: -1 - _best_score: {} - _class_map: null - _class_weight: null - _classes: null - _evals_result: {} - _n_classes: -1 - _n_features: -1 - _n_features_in: -1 - _objective: null - _other_params: - verbose: -1 - boosting_type: gbdt - class_weight: null - colsample_bytree: 1.0 - importance_type: split learning_rate: 0.02 - max_depth: -1 - min_child_samples: 20 - min_child_weight: 0.001 - min_split_gain: 0.0 + lgbmc: &id004 + name: LGBM Clas. + params: n_estimators: 500 - n_jobs: 1 - num_leaves: 31 - objective: null - random_state: null - reg_alpha: 0.0 - reg_lambda: 0.0 - subsample: 1.0 - subsample_for_bin: 200000 - subsample_freq: 0 - verbose: -1 + learning_rate: 0.02 +dml_parameters: + learners: + - ml_g: *id001 + ml_m: *id002 + - ml_g: *id003 + ml_m: *id004 score: - observational - experimental -simulation_parameters: - max_runtime: 19800 - n_jobs: -2 - random_seed: 42 - repetitions: 1000 + in_sample_normalization: + - true + - false +confidence_parameters: + level: + - 0.95 + - 0.9 From b5fe71f9143e1deadbc24c1de09df7ea377cfe15 Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Tue, 3 Jun 2025 10:19:30 +0200 Subject: [PATCH 13/16] fix rdrobust for fuzzy sim --- monte-cover/src/montecover/rdd/rdd.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/monte-cover/src/montecover/rdd/rdd.py b/monte-cover/src/montecover/rdd/rdd.py index b01caa0..8c36d80 100644 --- a/monte-cover/src/montecover/rdd/rdd.py +++ b/monte-cover/src/montecover/rdd/rdd.py @@ -114,7 +114,11 @@ def _rdrobust_benchmark(self, dml_data, dml_params, i_rep): benchmark_results_list = [] for level in self.confidence_parameters["level"]: - rd_model = rdrobust(y=Y, x=score, covs=Z, c=self.cutoff, level=level * 100) + if self.fuzzy: + D = dml_data.data[dml_data.d_cols] + rd_model = rdrobust(y=Y, x=score, fuzzy=D, covs=Z, c=self.cutoff, level=level * 100) + else: + rd_model = rdrobust(y=Y, x=score, covs=Z, c=self.cutoff, level=level * 100) coef_rd = rd_model.coef.loc["Robust", "Coeff"] ci_lower_rd = rd_model.ci.loc["Robust", "CI Lower"] ci_upper_rd = rd_model.ci.loc["Robust", "CI Upper"] From eef4c78c7520370659eee35795ed51dc611575cc Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Tue, 3 Jun 2025 10:19:58 +0200 Subject: [PATCH 14/16] update fuzzy config --- scripts/rdd/rdd_fuzzy_config.yml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/scripts/rdd/rdd_fuzzy_config.yml b/scripts/rdd/rdd_fuzzy_config.yml index 830515a..2e9cdc7 100644 --- a/scripts/rdd/rdd_fuzzy_config.yml +++ b/scripts/rdd/rdd_fuzzy_config.yml @@ -7,7 +7,7 @@ simulation_parameters: n_jobs: -2 dgp_parameters: - n_obs: [1000] # Sample size + n_obs: [2000] # Sample size fuzzy: [True] cutoff: [0.0] @@ -16,14 +16,16 @@ learner_definitions: lgbmr: &lgbmr name: "LGBM Regr." params: - n_estimators: 100 - learning_rate: 0.05 + n_estimators: 200 + learning_rate: 0.02 + max_depth: 5 lgbmc: &lgbmc name: "LGBM Clas." params: - n_estimators: 100 - learning_rate: 0.05 + n_estimators: 200 + learning_rate: 0.02 + max_depth: 5 global_linear: &global_linear name: "Global Linear" @@ -40,14 +42,16 @@ learner_definitions: stacked_reg: &stacked_reg name: "Stacked Regr." params: - n_estimators: 100 - learning_rate: 0.05 + n_estimators: 200 + learning_rate: 0.02 + max_depth: 5 stacked_cls: &stacked_cls name: "Stacked Clas." params: - n_estimators: 100 - learning_rate: 0.05 + n_estimators: 200 + learning_rate: 0.02 + max_depth: 5 dml_parameters: fs_specification: ["cutoff", "cutoff and score", "interacted cutoff and score"] From 888a88802750364683715c8daf95b331438ece8f Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Tue, 3 Jun 2025 10:20:11 +0200 Subject: [PATCH 15/16] rerun fuzzy sim --- results/rdd/rdd_fuzzy_config.yml | 22 +++++++------ results/rdd/rdd_fuzzy_coverage.csv | 52 +++++++++++++++--------------- results/rdd/rdd_fuzzy_metadata.csv | 2 +- 3 files changed, 40 insertions(+), 36 deletions(-) diff --git a/results/rdd/rdd_fuzzy_config.yml b/results/rdd/rdd_fuzzy_config.yml index d8b09c0..1c010bd 100644 --- a/results/rdd/rdd_fuzzy_config.yml +++ b/results/rdd/rdd_fuzzy_config.yml @@ -5,7 +5,7 @@ simulation_parameters: n_jobs: -2 dgp_parameters: n_obs: - - 1000 + - 2000 fuzzy: - true cutoff: @@ -14,13 +14,15 @@ learner_definitions: lgbmr: &id001 name: LGBM Regr. params: - n_estimators: 100 - learning_rate: 0.05 + n_estimators: 200 + learning_rate: 0.02 + max_depth: 5 lgbmc: &id002 name: LGBM Clas. params: - n_estimators: 100 - learning_rate: 0.05 + n_estimators: 200 + learning_rate: 0.02 + max_depth: 5 global_linear: &id003 name: Global Linear global_logistic: &id004 @@ -32,13 +34,15 @@ learner_definitions: stacked_reg: &id007 name: Stacked Regr. params: - n_estimators: 100 - learning_rate: 0.05 + n_estimators: 200 + learning_rate: 0.02 + max_depth: 5 stacked_cls: &id008 name: Stacked Clas. params: - n_estimators: 100 - learning_rate: 0.05 + n_estimators: 200 + learning_rate: 0.02 + max_depth: 5 dml_parameters: fs_specification: - cutoff diff --git a/results/rdd/rdd_fuzzy_coverage.csv b/results/rdd/rdd_fuzzy_coverage.csv index 71a7b18..a9a6dea 100644 --- a/results/rdd/rdd_fuzzy_coverage.csv +++ b/results/rdd/rdd_fuzzy_coverage.csv @@ -1,27 +1,27 @@ Method,fs_specification,Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -RDFlex,cutoff,Global Linear,Global Logistic,0.9,0.9363333333333334,61.21542033320797,6.094527516865915,1000 -RDFlex,cutoff,Global Linear,Global Logistic,0.95,0.972,72.942672336101,6.094527516865915,1000 -RDFlex,cutoff,LGBM Regr.,LGBM Clas.,0.9,0.9513333333333334,8.933409181593555,1.5599808778760562,1000 -RDFlex,cutoff,LGBM Regr.,LGBM Clas.,0.95,0.9876666666666666,10.644813598115608,1.5599808778760562,1000 -RDFlex,cutoff,Linear,Logistic,0.9,0.939,34.93607282259327,5.6843493908942815,1000 -RDFlex,cutoff,Linear,Logistic,0.95,0.973,41.62889838438588,5.6843493908942815,1000 -RDFlex,cutoff,Stacked Regr.,Stacked Clas.,0.9,0.9503333333333334,6.7442427655640484,1.3456834189094324,1000 -RDFlex,cutoff,Stacked Regr.,Stacked Clas.,0.95,0.9853333333333334,8.036260921282771,1.3456834189094324,1000 -RDFlex,cutoff and score,Global Linear,Global Logistic,0.9,0.9346666666666666,4016.9877724310254,50.8166330459716,1000 -RDFlex,cutoff and score,Global Linear,Global Logistic,0.95,0.9713333333333334,4786.536158171398,50.8166330459716,1000 -RDFlex,cutoff and score,LGBM Regr.,LGBM Clas.,0.9,0.951,9.065887957351245,1.5153501497320774,1000 -RDFlex,cutoff and score,LGBM Regr.,LGBM Clas.,0.95,0.9873333333333334,10.802671795919059,1.5153501497320774,1000 -RDFlex,cutoff and score,Linear,Logistic,0.9,0.94,335.8084886364535,7.708410484814828,1000 -RDFlex,cutoff and score,Linear,Logistic,0.95,0.9746666666666667,400.1404943551824,7.708410484814828,1000 -RDFlex,cutoff and score,Stacked Regr.,Stacked Clas.,0.9,0.9496666666666667,7.718643198281012,1.36917558119484,1000 -RDFlex,cutoff and score,Stacked Regr.,Stacked Clas.,0.95,0.9866666666666666,9.1973306501346,1.36917558119484,1000 -RDFlex,interacted cutoff and score,Global Linear,Global Logistic,0.9,0.939,49.6899424879847,5.516111563740509,1000 -RDFlex,interacted cutoff and score,Global Linear,Global Logistic,0.95,0.9723333333333334,59.20921842195623,5.516111563740509,1000 -RDFlex,interacted cutoff and score,LGBM Regr.,LGBM Clas.,0.9,0.949,8.869451650244981,1.5644296749591,1000 -RDFlex,interacted cutoff and score,LGBM Regr.,LGBM Clas.,0.95,0.9866666666666666,10.568603498974065,1.5644296749591,1000 -RDFlex,interacted cutoff and score,Linear,Logistic,0.9,0.939,165.6185238963947,6.482221436992595,1000 -RDFlex,interacted cutoff and score,Linear,Logistic,0.95,0.9733333333333334,197.34664330663676,6.482221436992595,1000 -RDFlex,interacted cutoff and score,Stacked Regr.,Stacked Clas.,0.9,0.953,12.965304115439578,1.8427674267462426,1000 -RDFlex,interacted cutoff and score,Stacked Regr.,Stacked Clas.,0.95,0.9833333333333334,15.449112734710363,1.8427674267462426,1000 -rdrobust,cutoff,Linear,Logistic,0.9,0.66,2.2540261137473974,0.9256316634723921,1000 -rdrobust,cutoff,Linear,Logistic,0.95,0.768,2.685837773507866,0.9256316634723921,1000 +RDFlex,cutoff,Global Linear,Global Logistic,0.9,0.8943333333333334,9.45138989718295,2.373538922365825,1000 +RDFlex,cutoff,Global Linear,Global Logistic,0.95,0.9466666666666667,11.26202568957878,2.373538922365825,1000 +RDFlex,cutoff,LGBM Regr.,LGBM Clas.,0.9,0.9106666666666666,2.098970644289801,0.5229709395139468,1000 +RDFlex,cutoff,LGBM Regr.,LGBM Clas.,0.95,0.9606666666666667,2.5010777858935986,0.5229709395139468,1000 +RDFlex,cutoff,Linear,Logistic,0.9,0.898,9.475050602811462,2.38122109508206,1000 +RDFlex,cutoff,Linear,Logistic,0.95,0.9516666666666667,11.290219159271665,2.38122109508206,1000 +RDFlex,cutoff,Stacked Regr.,Stacked Clas.,0.9,0.9143333333333333,2.006664818935384,0.4926541115149405,1000 +RDFlex,cutoff,Stacked Regr.,Stacked Clas.,0.95,0.9643333333333334,2.3910886109946707,0.4926541115149405,1000 +RDFlex,cutoff and score,Global Linear,Global Logistic,0.9,0.896,9.45192675987028,2.3708700911814633,1000 +RDFlex,cutoff and score,Global Linear,Global Logistic,0.95,0.9483333333333334,11.262665400927295,2.3708700911814633,1000 +RDFlex,cutoff and score,LGBM Regr.,LGBM Clas.,0.9,0.9206666666666666,2.137979587602868,0.5310240118035273,1000 +RDFlex,cutoff and score,LGBM Regr.,LGBM Clas.,0.95,0.9706666666666667,2.547559808801788,0.5310240118035273,1000 +RDFlex,cutoff and score,Linear,Logistic,0.9,0.8993333333333333,9.431787596286874,2.370891486466498,1000 +RDFlex,cutoff and score,Linear,Logistic,0.95,0.9506666666666667,11.238668107395837,2.370891486466498,1000 +RDFlex,cutoff and score,Stacked Regr.,Stacked Clas.,0.9,0.9206666666666666,2.021432595475123,0.4848163059226252,1000 +RDFlex,cutoff and score,Stacked Regr.,Stacked Clas.,0.95,0.968,2.408685502095107,0.4848163059226252,1000 +RDFlex,interacted cutoff and score,Global Linear,Global Logistic,0.9,0.8986666666666666,9.417430458001911,2.3532260082168035,1000 +RDFlex,interacted cutoff and score,Global Linear,Global Logistic,0.95,0.9506666666666667,11.221560521955702,2.3532260082168035,1000 +RDFlex,interacted cutoff and score,LGBM Regr.,LGBM Clas.,0.9,0.9203333333333333,2.1443333037007934,0.5343555930536144,1000 +RDFlex,interacted cutoff and score,LGBM Regr.,LGBM Clas.,0.95,0.9696666666666667,2.555130728496937,0.5343555930536144,1000 +RDFlex,interacted cutoff and score,Linear,Logistic,0.9,0.8983333333333333,9.463034507707663,2.36467701566644,1000 +RDFlex,interacted cutoff and score,Linear,Logistic,0.95,0.9486666666666667,11.275901098836155,2.36467701566644,1000 +RDFlex,interacted cutoff and score,Stacked Regr.,Stacked Clas.,0.9,0.9233333333333333,2.0607292105190114,0.5038394189514153,1000 +RDFlex,interacted cutoff and score,Stacked Regr.,Stacked Clas.,0.95,0.9763333333333334,2.455510307012918,0.5038394189514153,1000 +rdrobust,cutoff,Linear,Logistic,0.9,0.925,10.188022696252679,2.4726265121800406,1000 +rdrobust,cutoff,Linear,Logistic,0.95,0.97,12.13977780827851,2.4726265121800406,1000 diff --git a/results/rdd/rdd_fuzzy_metadata.csv b/results/rdd/rdd_fuzzy_metadata.csv index 1bee153..0c28df1 100644 --- a/results/rdd/rdd_fuzzy_metadata.csv +++ b/results/rdd/rdd_fuzzy_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.11.dev0,RDDCoverageSimulation,2025-06-02 14:19,128.11025975545246,3.12.3,scripts/rdd/rdd_fuzzy_config.yml +0.10.0,RDDCoverageSimulation,2025-06-03 10:17,24.518820464611053,3.12.9,scripts/rdd/rdd_fuzzy_config.yml From aed57136c69548786ed4a5222d92ab3745bae4f6 Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Tue, 3 Jun 2025 10:20:17 +0200 Subject: [PATCH 16/16] rerun did sim --- results/did/did_multi_detailed.csv | 96 ++++++++++++++-------------- results/did/did_multi_eventstudy.csv | 96 ++++++++++++++-------------- results/did/did_multi_group.csv | 96 ++++++++++++++-------------- results/did/did_multi_metadata.csv | 2 +- results/did/did_multi_time.csv | 96 ++++++++++++++-------------- 5 files changed, 193 insertions(+), 193 deletions(-) diff --git a/results/did/did_multi_detailed.csv b/results/did/did_multi_detailed.csv index 0792d64..ab7ab8b 100644 --- a/results/did/did_multi_detailed.csv +++ b/results/did/did_multi_detailed.csv @@ -1,49 +1,49 @@ Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.4084507042253521,0.6612522092326544,0.44973970228836385,0.10328638497652583,0.9935059914244001,213 -LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.49178403755868544,0.787930605834864,0.44973970228836385,0.1596244131455399,1.1014902175259917,213 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.5336463223787168,0.582766759983815,0.32299857116015934,0.18309859154929578,0.8961664076512703,213 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.6263693270735524,0.694409424790157,0.32299857116015934,0.28169014084507044,0.9867179947950292,213 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.9017996870109547,0.578959671088662,0.13751972629200543,0.8732394366197183,0.8919682440411882,213 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9491392801251956,0.6898729985707177,0.13751972629200543,0.9342723004694836,0.9817237170113741,213 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.4107981220657277,0.6610990972056453,0.4476286561990305,0.107981220657277,0.9924537064343187,213 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.5050860719874805,0.7877481616017598,0.4476286561990305,0.1643192488262911,1.099584135360425,213 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.5348200312989045,0.5827020149697208,0.32210086748537925,0.20187793427230047,0.8980725212627803,213 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.6255868544600939,0.6943322763474483,0.32210086748537925,0.29577464788732394,0.9881288275749126,213 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.9021909233176838,0.5789579680053694,0.13548606467162275,0.892018779342723,0.8906297555939263,213 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9518779342723005,0.6898709692218071,0.13548606467162275,0.9389671361502347,0.9807905334864873,213 -LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9197965571205008,2.6943514650786105,0.6839666644937138,0.9765258215962441,4.190030688848515,213 -LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9655712050078247,3.210517791199548,0.6839666644937138,0.9859154929577465,4.601967336099581,213 -LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9194053208137716,3.4922540819240244,0.9237972600914225,0.9812206572769953,5.388123646746446,213 -LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9698748043818467,4.16127740078602,0.9237972600914225,1.0,5.930060622023973,213 -LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9354460093896714,2.1911606588763415,0.5011619074030041,0.9624413145539906,3.4237130298632317,213 -LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.969092331768388,2.6109289637511237,0.5011619074030041,0.9953051643192489,3.7555769901088496,213 -LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9139280125195618,1.109267010415937,0.2755940893225088,0.8685446009389671,1.7291751019525257,213 -LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.954225352112676,1.321773168159112,0.2755940893225088,0.9342723004694836,1.8979923185233545,213 -LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9205790297339593,1.4158326633799614,0.33401929336661684,0.9107981220657277,2.1920950306133054,213 -LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9616588419405321,1.6870686746170975,0.33401929336661684,0.9389671361502347,2.408946885913222,213 -LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9123630672926448,1.0199912152288533,0.24510901682198524,0.9248826291079812,1.5958111429274424,213 -LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9561815336463223,1.2153944968957262,0.24510901682198524,0.971830985915493,1.7484330943458928,213 -Linear,Logistic,experimental,False,1,0.9,0.8450704225352113,0.29447476044239895,0.08354776001109418,0.7276995305164319,0.4596087013280646,213 -Linear,Logistic,experimental,False,1,0.95,0.9084507042253521,0.35088831940192455,0.08354776001109418,0.8075117370892019,0.5045368833572496,213 -Linear,Logistic,experimental,False,4,0.9,0.3341158059467919,0.9748301672712222,0.7902188560164397,0.056338028169014086,1.4111319523263854,213 -Linear,Logistic,experimental,False,4,0.95,0.4017996870109546,1.1615817891564408,0.7902188560164397,0.07042253521126761,1.5730924328858458,213 -Linear,Logistic,experimental,False,6,0.9,0.9080594679186228,0.9825593405176763,0.23361443805782198,0.9154929577464789,1.4168592399518378,213 -Linear,Logistic,experimental,False,6,0.95,0.956964006259781,1.1707916671328766,0.23361443805782198,0.9483568075117371,1.5807909190964573,213 -Linear,Logistic,experimental,True,1,0.9,0.8438967136150235,0.29448998280268696,0.08353477895677355,0.7183098591549296,0.45943729886831725,213 -Linear,Logistic,experimental,True,1,0.95,0.9088419405320814,0.35090645796297054,0.08353477895677355,0.812206572769953,0.5041114400484898,213 -Linear,Logistic,experimental,True,4,0.9,0.3333333333333333,0.9747975428898821,0.7903965236641811,0.056338028169014086,1.4119051235101507,213 -Linear,Logistic,experimental,True,4,0.95,0.4033646322378717,1.161542914808355,0.7903965236641811,0.07981220657276995,1.5743781294233699,213 -Linear,Logistic,experimental,True,6,0.9,0.9092331768388106,0.9825623304514758,0.2341476257220959,0.92018779342723,1.41979846440565,213 -Linear,Logistic,experimental,True,6,0.95,0.9553990610328639,1.1707952298587427,0.2341476257220959,0.9530516431924883,1.5817945290736959,213 -Linear,Logistic,observational,False,1,0.9,0.894757433489828,0.3183557829503854,0.07809759992541833,0.892018779342723,0.49488785729946616,213 -Linear,Logistic,observational,False,1,0.95,0.9483568075117371,0.3793443128488264,0.07809759992541833,0.9436619718309859,0.5435074166885125,213 -Linear,Logistic,observational,False,4,0.9,0.44405320813771515,1.2415493083413676,0.7711800746228612,0.19718309859154928,1.774858650110625,213 -Linear,Logistic,observational,False,4,0.95,0.5453834115805947,1.4793972481853468,0.7711800746228612,0.2676056338028169,1.9858757227772874,213 -Linear,Logistic,observational,False,6,0.9,0.9115805946791862,1.0304949773365348,0.24398615030125753,0.92018779342723,1.4858759045886074,213 -Linear,Logistic,observational,False,6,0.95,0.9557902973395932,1.2279105014178964,0.24398615030125753,0.9577464788732394,1.6602060040028728,213 -Linear,Logistic,observational,True,1,0.9,0.892018779342723,0.3163792430260323,0.07808090534806102,0.8873239436619719,0.4927677081751961,213 -Linear,Logistic,observational,True,1,0.95,0.9464006259780908,0.37698912026374665,0.07808090534806102,0.9530516431924883,0.5411219121582408,213 -Linear,Logistic,observational,True,4,0.9,0.4409233176838811,1.2414842562609323,0.7671739690013839,0.19718309859154928,1.7725218888802696,213 -Linear,Logistic,observational,True,4,0.95,0.5410798122065728,1.4793197338505248,0.7671739690013839,0.28169014084507044,1.9817316211484592,213 -Linear,Logistic,observational,True,6,0.9,0.9061032863849765,1.027459591935346,0.24734972591149948,0.9154929577464789,1.4817614944804902,213 -Linear,Logistic,observational,True,6,0.95,0.9546165884194053,1.2242936166276337,0.24734972591149948,0.9436619718309859,1.6552966469110102,213 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.4005833333333333,0.668417462452362,0.4514209317520677,0.069,1.001536224942987,1000 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.49541666666666667,0.7964685316542967,0.4514209317520677,0.119,1.1096223780406622,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.525,0.5836476641555988,0.3357039587486384,0.192,0.8984549944516823,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.6135,0.6954590868526271,0.3357039587486384,0.266,0.9889489428407441,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.8913333333333334,0.5798205422267835,0.1436401623087172,0.896,0.8921080477762784,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9469166666666666,0.6908987898012575,0.1436401623087172,0.95,0.9823087136174479,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.403,0.668486737121192,0.4516172220554326,0.064,1.0020517044274269,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.4925,0.7965510775135288,0.4516172220554326,0.118,1.110697612751064,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.5244166666666666,0.5834979623893575,0.33563072303718966,0.202,0.8980797806262651,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.6114166666666666,0.6952807061958639,0.33563072303718966,0.275,0.9886661012047989,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.8903333333333334,0.5798432460053057,0.1452858251598464,0.888,0.8930518639150258,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9449166666666666,0.690925843021974,0.1452858251598464,0.942,0.9832578157026843,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9115,2.7279550989921355,0.7081663387461643,0.951,4.239219420515267,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9635833333333333,3.250558990696578,0.7081663387461643,0.988,4.6533388042213675,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9074166666666666,3.5225937797559883,0.9757809691235549,0.97,5.426540882685481,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9669166666666666,4.19742938055986,0.9757809691235549,0.992,5.969933156329405,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9271666666666666,2.1769181957500305,0.5048068220768145,0.971,3.397149166606052,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9709166666666667,2.5939580221905385,0.5048068220768145,0.996,3.7281189530691132,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.90375,1.1192914856600868,0.27862527337620824,0.928,1.7467988922140234,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9589166666666666,1.3337180671583386,0.27862527337620824,0.967,1.914692647031146,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9199166666666666,1.4197310386522546,0.32827809352307774,0.935,2.19395277603094,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9664166666666666,1.6917138752639644,0.32827809352307774,0.968,2.4119391624873256,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9005,1.0294284373000027,0.2502841114417097,0.917,1.6092703629556147,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9533333333333334,1.226639640579326,0.2502841114417097,0.971,1.7641639749309233,1000 +Linear,Logistic,experimental,False,1,0.9,0.8521666666666666,0.2947207725579002,0.08125759297583215,0.769,0.45936907943411753,1000 +Linear,Logistic,experimental,False,1,0.95,0.91225,0.3511814609181314,0.08125759297583215,0.857,0.5043035205016457,1000 +Linear,Logistic,experimental,False,4,0.9,0.3186666666666667,0.9766434317991218,0.8093782242316307,0.041,1.4141166689592581,1000 +Linear,Logistic,experimental,False,4,0.95,0.393,1.1637424271067696,0.8093782242316307,0.079,1.5775150069804904,1000 +Linear,Logistic,experimental,False,6,0.9,0.8959166666666666,0.9840677614600047,0.24403762906388674,0.889,1.4217412744627138,1000 +Linear,Logistic,experimental,False,6,0.95,0.9463333333333334,1.1725890615466086,0.24403762906388674,0.94,1.585826837431638,1000 +Linear,Logistic,experimental,True,1,0.9,0.8528333333333333,0.29471605836940884,0.08132561217659459,0.764,0.45923649733243366,1000 +Linear,Logistic,experimental,True,1,0.95,0.9120833333333334,0.351175843616076,0.08132561217659459,0.855,0.5041154397102058,1000 +Linear,Logistic,experimental,True,4,0.9,0.3188333333333333,0.9765337199818319,0.8092231167974461,0.041,1.4131241013020541,1000 +Linear,Logistic,experimental,True,4,0.95,0.39458333333333334,1.1636116974132316,0.8092231167974461,0.076,1.5757045631534141,1000 +Linear,Logistic,experimental,True,6,0.9,0.8968333333333334,0.984151005519869,0.24420883449452355,0.889,1.4208221899399172,1000 +Linear,Logistic,experimental,True,6,0.95,0.94675,1.1726882529619342,0.24420883449452355,0.936,1.5847814464063683,1000 +Linear,Logistic,observational,False,1,0.9,0.9001666666666667,0.3180779030235929,0.0773596882018309,0.88,0.49501077303673774,1000 +Linear,Logistic,observational,False,1,0.95,0.94775,0.3790131984933507,0.0773596882018309,0.947,0.5435398448787098,1000 +Linear,Logistic,observational,False,4,0.9,0.4245,1.237493689045589,0.7914353096522312,0.18,1.76761883251908,1000 +Linear,Logistic,observational,False,4,0.95,0.5209166666666666,1.4745646797278944,0.7914353096522312,0.275,1.9755226904473104,1000 +Linear,Logistic,observational,False,6,0.9,0.8929166666666666,1.0255283640310429,0.255700122939389,0.895,1.4816116459637685,1000 +Linear,Logistic,observational,False,6,0.95,0.9455,1.221992416644637,0.255700122939389,0.934,1.653527148130022,1000 +Linear,Logistic,observational,True,1,0.9,0.8965,0.31619901580629955,0.07736324119662329,0.885,0.49203725267016324,1000 +Linear,Logistic,observational,True,1,0.95,0.9463333333333334,0.3767743662856892,0.07736324119662329,0.947,0.5403636983384068,1000 +Linear,Logistic,observational,True,4,0.9,0.4231666666666667,1.2357474079437374,0.7915034422252056,0.193,1.7654458413619594,1000 +Linear,Logistic,observational,True,4,0.95,0.5213333333333334,1.472483857452629,0.7915034422252056,0.277,1.9735390159048785,1000 +Linear,Logistic,observational,True,6,0.9,0.8928333333333334,1.0212995120870143,0.25694449841811784,0.89,1.4744168456619084,1000 +Linear,Logistic,observational,True,6,0.95,0.9460833333333334,1.216953428755113,0.25694449841811784,0.933,1.6462184670600197,1000 diff --git a/results/did/did_multi_eventstudy.csv b/results/did/did_multi_eventstudy.csv index 2498639..1c294b6 100644 --- a/results/did/did_multi_eventstudy.csv +++ b/results/did/did_multi_eventstudy.csv @@ -1,49 +1,49 @@ Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.2793427230046948,0.6538940149561205,0.5211711102717896,0.06103286384976526,0.86055821069107,213 -LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.3528951486697966,0.7791627765660771,0.5211711102717896,0.1267605633802817,0.9765429620402541,213 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.39358372456964,0.541925194741234,0.3658723164598346,0.14084507042253522,0.7380373644216336,213 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.4694835680751174,0.6457436981649495,0.3658723164598346,0.30985915492957744,0.8302776266426216,213 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.9014084507042254,0.5382900596052185,0.12834769643838123,0.9014084507042254,0.7329552647921785,213 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9499217527386542,0.6414121674872129,0.12834769643838123,0.9530516431924883,0.8240621744454719,213 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.2902973395931142,0.653663670745508,0.5177678460962447,0.07511737089201878,0.8606157129718126,213 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.36932707355242567,0.7788883045100532,0.5177678460962447,0.14084507042253522,0.978192241517096,213 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.3763693270735525,0.5420271299376794,0.36469586751312316,0.18779342723004694,0.7374141258484136,213 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.4647887323943662,0.645865161443211,0.36469586751312316,0.27230046948356806,0.8307165715799049,213 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.9194053208137716,0.5383523983929166,0.12524713080884226,0.9248826291079812,0.7331206282884405,213 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9483568075117371,0.6414864487343258,0.12524713080884226,0.9389671361502347,0.8244543573667497,213 -LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9131455399061033,2.635762776399446,0.6863198917243725,0.9389671361502347,3.6611895504458305,213 -LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9608763693270735,3.1407050626801003,0.6863198917243725,0.9812206572769953,4.095116662293061,213 -LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9107981220657277,3.5476693552380327,0.9618718220501209,0.9577464788732394,4.871536434987288,213 -LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9733959311424101,4.227308771668667,0.9618718220501209,0.9765258215962441,5.471362486927891,213 -LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.94679186228482,2.048525808104845,0.4424199042553949,0.9530516431924883,2.8547596856961075,213 -LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9780907668231612,2.4409690561510136,0.4424199042553949,0.9859154929577465,3.191610282696814,213 -LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9178403755868545,1.0506825752853373,0.25838130000929665,0.9248826291079812,1.4648011676731838,213 -LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9624413145539906,1.2519655080553924,0.25838130000929665,0.9577464788732394,1.6360312921813787,213 -LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9225352112676056,1.3999042833283855,0.32349351954396005,0.9248826291079812,1.9287116595218243,213 -LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9679186228482003,1.6680888391340973,0.32349351954396005,0.9624413145539906,2.1548062661512533,213 -LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9194053208137716,0.941095371039425,0.21448858413067756,0.9530516431924883,1.3188087131097894,213 -LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9710485133020345,1.1213843001174537,0.21448858413067756,0.9812206572769953,1.4720050449124056,213 -Linear,Logistic,experimental,False,1,0.9,0.8176838810641627,0.2097616817290772,0.06354647829956948,0.7511737089201878,0.29957865791433536,213 -Linear,Logistic,experimental,False,1,0.95,0.888888888888889,0.24994646015251432,0.06354647829956948,0.8356807511737089,0.3324902033558004,213 -Linear,Logistic,experimental,False,4,0.9,0.20735524256651017,0.9731315969907915,0.9247823933778377,0.051643192488262914,1.2553334876224438,213 -Linear,Logistic,experimental,False,4,0.95,0.26682316118935834,1.1595578178313908,0.9247823933778377,0.06572769953051644,1.427245013596636,213 -Linear,Logistic,experimental,False,6,0.9,0.9162754303599373,0.982583831624362,0.2340222441493123,0.9107981220657277,1.2642250097051084,213 -Linear,Logistic,experimental,False,6,0.95,0.9538341158059468,1.1708208500864592,0.2340222441493123,0.9436619718309859,1.440074754119655,213 -Linear,Logistic,experimental,True,1,0.9,0.8184663536776213,0.209784288558112,0.06346289348873158,0.7511737089201878,0.2994094620604152,213 -Linear,Logistic,experimental,True,1,0.95,0.8896713615023474,0.24997339785079145,0.06346289348873158,0.8497652582159625,0.33304580863290817,213 -Linear,Logistic,experimental,True,4,0.9,0.21048513302034427,0.9730155853528542,0.9248338055294343,0.051643192488262914,1.2540423515599672,213 -Linear,Logistic,experimental,True,4,0.95,0.2699530516431925,1.1594195814385477,0.9248338055294343,0.07042253521126761,1.4266453422038732,213 -Linear,Logistic,experimental,True,6,0.9,0.9139280125195618,0.9825202138375453,0.23446937547502114,0.9061032863849765,1.2635545418793521,213 -Linear,Logistic,experimental,True,6,0.95,0.9522691705790298,1.1707450448178967,0.23446937547502114,0.9389671361502347,1.4364612584863943,213 -Linear,Logistic,observational,False,1,0.9,0.9100156494522692,0.2256652512730398,0.05275781831070359,0.9248826291079812,0.3210767224862521,213 -Linear,Logistic,observational,False,1,0.95,0.960093896713615,0.26889673209225234,0.05275781831070359,0.9483568075117371,0.357572619911957,213 -Linear,Logistic,observational,False,4,0.9,0.3325508607198748,1.2935393301274787,0.9012986658762238,0.20187793427230047,1.6449223740918557,213 -Linear,Logistic,observational,False,4,0.95,0.4460093896713615,1.5413471801346634,0.9012986658762238,0.2863849765258216,1.8802622354415948,213 -Linear,Logistic,observational,False,6,0.9,0.9123630672926448,1.0362716142235429,0.2466341904936733,0.9248826291079812,1.328413177022464,213 -Linear,Logistic,observational,False,6,0.95,0.960093896713615,1.2347937888209737,0.2466341904936733,0.9483568075117371,1.5124625048982703,213 -Linear,Logistic,observational,True,1,0.9,0.906885758998435,0.22449816162535258,0.05285134622236858,0.9107981220657277,0.32006341717518294,213 -Linear,Logistic,observational,True,1,0.95,0.9561815336463223,0.267506059002127,0.05285134622236858,0.9483568075117371,0.355619479598059,213 -Linear,Logistic,observational,True,4,0.9,0.3380281690140845,1.2948063187256469,0.8972215616534344,0.1784037558685446,1.6460425935700733,213 -Linear,Logistic,observational,True,4,0.95,0.430359937402191,1.5428568901663307,0.8972215616534344,0.3051643192488263,1.8803448054869478,213 -Linear,Logistic,observational,True,6,0.9,0.9123630672926448,1.0324801100143304,0.2506321361153765,0.9014084507042254,1.3236751031745002,213 -Linear,Logistic,observational,True,6,0.95,0.9499217527386542,1.2302759329002244,0.2506321361153765,0.9530516431924883,1.510018818061999,213 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.26816666666666666,0.6628004801492844,0.5217146610039974,0.063,0.8710835347098573,1000 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.356,0.7897754844217385,0.5217146610039974,0.106,0.9881445074099509,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.3793333333333333,0.5430545733393948,0.3833854158756925,0.176,0.73981228970138,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.4698333333333333,0.6470894357680017,0.3833854158756925,0.246,0.833246241332189,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.8915,0.5395169482604701,0.13534305762469595,0.897,0.7348860485385167,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9451666666666666,0.6428740954897616,0.13534305762469595,0.953,0.8259292882870245,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.265,0.6629226425912405,0.5221091420199984,0.062,0.8722109625410929,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.3551666666666667,0.7899210499496214,0.5221091420199984,0.114,0.9882984410974919,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.3763333333333333,0.5429965130173356,0.3828566010032573,0.182,0.739132671094994,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.4711666666666667,0.6470202526271421,0.3828566010032573,0.262,0.8314461004262811,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.8931666666666667,0.5395318624534728,0.13758915825744059,0.893,0.7341757740643098,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9463333333333334,0.6428918668468384,0.13758915825744059,0.946,0.8261153685402325,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.907,2.635755403048029,0.6906219348022667,0.931,3.658094889963508,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9618333333333333,3.140696276789984,0.6906219348022667,0.969,4.09528459799626,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.8978333333333334,3.5932112684720123,1.044176882449884,0.953,4.920160668033086,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9656666666666667,4.2815753083763015,1.044176882449884,0.982,5.521039436439466,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9356666666666666,2.037043783641883,0.45673572364777687,0.967,2.8380713672583395,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9768333333333333,2.4272873801354264,0.45673572364777687,0.99,3.173284984968741,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9186666666666666,1.058073826869023,0.2586744612407186,0.924,1.4747488796573536,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9613333333333334,1.2607727275351879,0.2586744612407186,0.965,1.649868845892826,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9368333333333334,1.4039435411486991,0.3103897362727539,0.943,1.9287327010063178,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9718333333333333,1.6729019116910515,0.3103897362727539,0.973,2.1651520912520748,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9153333333333333,0.9537908163093017,0.2248364423186676,0.927,1.3333350854360233,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9601666666666666,1.1365118562044807,0.2248364423186676,0.974,1.4916507037488054,1000 +Linear,Logistic,experimental,False,1,0.9,0.8085,0.21012919637470698,0.06454170881550668,0.737,0.3000298498259047,1000 +Linear,Logistic,experimental,False,1,0.95,0.8826666666666666,0.2503843808631616,0.06454170881550668,0.83,0.33342829695147896,1000 +Linear,Logistic,experimental,False,4,0.9,0.20066666666666666,0.9748536890898165,0.9456802642466463,0.041,1.2574525229653895,1000 +Linear,Logistic,experimental,False,4,0.95,0.26666666666666666,1.161609817132025,0.9456802642466463,0.074,1.4305039091925136,1000 +Linear,Logistic,experimental,False,6,0.9,0.8906666666666666,0.984266405297522,0.2447629940182398,0.885,1.2654037292644829,1000 +Linear,Logistic,experimental,False,6,0.95,0.9428333333333334,1.1728257602782801,0.2447629940182398,0.939,1.4411880110797295,1000 +Linear,Logistic,experimental,True,1,0.9,0.8095,0.21012765534784542,0.0646279391502723,0.733,0.29993071695819135,1000 +Linear,Logistic,experimental,True,1,0.95,0.8823333333333334,0.25038254461639875,0.0646279391502723,0.831,0.3331089494375711,1000 +Linear,Logistic,experimental,True,4,0.9,0.201,0.9747048561891234,0.9456904408930068,0.042,1.2571661797897207,1000 +Linear,Logistic,experimental,True,4,0.95,0.26666666666666666,1.1614324717924194,0.9456904408930068,0.075,1.4292891253140116,1000 +Linear,Logistic,experimental,True,6,0.9,0.8928333333333334,0.9843540987911897,0.24472186099367751,0.882,1.2667938573542303,1000 +Linear,Logistic,experimental,True,6,0.95,0.942,1.1729302535209924,0.24472186099367751,0.938,1.4425417792737238,1000 +Linear,Logistic,observational,False,1,0.9,0.8958333333333334,0.22595429878581444,0.05556582705974583,0.883,0.32193445273348237,1000 +Linear,Logistic,observational,False,1,0.95,0.9468333333333334,0.26924115344718413,0.05556582705974583,0.944,0.35775078341337774,1000 +Linear,Logistic,observational,False,4,0.9,0.325,1.2886928093204373,0.9238068984287002,0.175,1.6397882387716314,1000 +Linear,Logistic,observational,False,4,0.95,0.41933333333333334,1.535572194399471,0.9238068984287002,0.26,1.8701388333818019,1000 +Linear,Logistic,observational,False,6,0.9,0.8893333333333334,1.0302622288172734,0.2580572852924157,0.882,1.322817867501306,1000 +Linear,Logistic,observational,False,6,0.95,0.9411666666666666,1.2276331644514131,0.2580572852924157,0.934,1.5071984875399793,1000 +Linear,Logistic,observational,True,1,0.9,0.8921666666666667,0.22482298708967266,0.0554326928262382,0.882,0.3206665757666365,1000 +Linear,Logistic,observational,True,1,0.95,0.9446666666666667,0.2678931124158151,0.0554326928262382,0.941,0.3562766006057057,1000 +Linear,Logistic,observational,True,4,0.9,0.3243333333333333,1.2866810812005252,0.9246774960344024,0.177,1.6374108099983478,1000 +Linear,Logistic,observational,True,4,0.95,0.41833333333333333,1.5331750724932363,0.9246774960344024,0.258,1.869004453360348,1000 +Linear,Logistic,observational,True,6,0.9,0.8853333333333334,1.0254952185219397,0.25982703889124587,0.889,1.318157054156258,1000 +Linear,Logistic,observational,True,6,0.95,0.9436666666666667,1.2219529213345213,0.25982703889124587,0.94,1.502504660229367,1000 diff --git a/results/did/did_multi_group.csv b/results/did/did_multi_group.csv index de75e59..e3035bc 100644 --- a/results/did/did_multi_group.csv +++ b/results/did/did_multi_group.csv @@ -1,49 +1,49 @@ Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.38341158059467917,0.7010247560032261,0.5045579001393814,0.06572769953051644,0.8747944725382377,213 -LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.44913928012519555,0.8353225183834784,0.5045579001393814,0.13145539906103287,0.9956978967225759,213 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.514866979655712,0.6082806742773996,0.35754266882547,0.18309859154929578,0.7739715228608116,213 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.5884194053208137,0.7248111288084945,0.35754266882547,0.29107981220657275,0.878325999451341,213 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.9045383411580594,0.6032942628400023,0.1416087698948157,0.9061032863849765,0.7672596153857486,213 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9577464788732394,0.7188694531060122,0.1416087698948157,0.9483568075117371,0.8700719219107395,213 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.38028169014084506,0.7006624811428777,0.5027623497763666,0.08450704225352113,0.8749076136062413,213 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.45383411580594674,0.8348908412620905,0.5027623497763666,0.14084507042253522,0.9960800157981575,213 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.49765258215962443,0.6078684171461785,0.3564700998360187,0.1784037558685446,0.7724813665598308,213 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.5915492957746479,0.7243198941379295,0.3564700998360187,0.28169014084507044,0.8768342044058519,213 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.9092331768388106,0.6035874382384104,0.13992167424040736,0.9295774647887324,0.7668293165413835,213 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9577464788732394,0.7192187931400541,0.13992167424040736,0.9483568075117371,0.8695905007790928,213 -LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9233176838810642,2.6401625405977405,0.6680316774131236,0.9530516431924883,3.348805972231518,213 -LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9687010954616588,3.1459477050817273,0.6680316774131236,0.9812206572769953,3.791143326987578,213 -LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9123630672926448,3.4982482043306202,0.9449299713795152,0.9248826291079812,4.416321409189137,213 -LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9577464788732394,4.1684198381697195,0.9449299713795152,0.9812206572769953,5.007980964364257,213 -LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9389671361502347,2.1556775041672305,0.46296654578958996,0.9483568075117371,2.739619518142511,213 -LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9733959311424101,2.568648177091331,0.46296654578958996,0.9906103286384976,3.1020251380422854,213 -LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9264475743348983,1.0864056229119294,0.2572919037765861,0.9389671361502347,1.3803957820627104,213 -LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9624413145539906,1.2945321447572218,0.2572919037765861,0.9624413145539906,1.5630008344403068,213 -LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9405320813771518,1.4335989322710556,0.318773509562249,0.9577464788732394,1.8127247140484581,213 -LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9765258215962441,1.7082384897274767,0.318773509562249,0.9859154929577465,2.0590624139128217,213 -LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.917057902973396,1.015686570779001,0.23606871293652523,0.9483568075117371,1.2952585886030479,213 -LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9640062597809077,1.2102651966651654,0.23606871293652523,0.9812206572769953,1.4663039002495655,213 -Linear,Logistic,experimental,False,1,0.9,0.8169014084507042,0.263859732601683,0.08071876563864448,0.7746478873239436,0.33907438784389654,213 -Linear,Logistic,experimental,False,1,0.95,0.895148669796557,0.31440826368735936,0.08071876563864448,0.8544600938967136,0.38284950361525444,213 -Linear,Logistic,experimental,False,4,0.9,0.3302034428794992,1.077782919659112,0.8937614998777182,0.051643192488262914,1.3561544020534126,213 -Linear,Logistic,experimental,False,4,0.95,0.39749608763693267,1.2842575601084827,0.8937614998777182,0.07511737089201878,1.5423676656010097,213 -Linear,Logistic,experimental,False,6,0.9,0.9092331768388106,1.084908190453934,0.25576007865798384,0.9295774647887324,1.365127434841,213 -Linear,Logistic,experimental,False,6,0.95,0.9593114241001566,1.2927478439301676,0.25576007865798384,0.9577464788732394,1.55476043284721,213 -Linear,Logistic,experimental,True,1,0.9,0.8106416275430359,0.26388645094216934,0.08082008766978688,0.7793427230046949,0.33860699865003,213 -Linear,Logistic,experimental,True,1,0.95,0.8982785602503913,0.31444010055370525,0.08082008766978688,0.8544600938967136,0.38363187019926315,213 -Linear,Logistic,experimental,True,4,0.9,0.32707355242566505,1.0776870925219295,0.8937933391766094,0.051643192488262914,1.3587107289972284,213 -Linear,Logistic,experimental,True,4,0.95,0.40219092331768386,1.284143375031743,0.8937933391766094,0.07981220657276995,1.5423186997471712,213 -Linear,Logistic,experimental,True,6,0.9,0.9123630672926448,1.0847636730976646,0.25673459846880975,0.9295774647887324,1.3684316527340743,213 -Linear,Logistic,experimental,True,6,0.95,0.9530516431924883,1.2925756408789129,0.25673459846880975,0.9530516431924883,1.552030854337275,213 -Linear,Logistic,observational,False,1,0.9,0.895148669796557,0.2840761342590122,0.06969384593021301,0.9154929577464789,0.36506933928947094,213 -Linear,Logistic,observational,False,1,0.95,0.9577464788732394,0.3384975920604852,0.06969384593021301,0.9483568075117371,0.41184024131390085,213 -Linear,Logistic,observational,False,4,0.9,0.42879499217527384,1.3815569998881347,0.8834464565663525,0.19718309859154928,1.7278992279793388,213 -Linear,Logistic,observational,False,4,0.95,0.5226917057902973,1.646226702486907,0.8834464565663525,0.2863849765258216,1.9664187739003602,213 -Linear,Logistic,observational,False,6,0.9,0.9139280125195618,1.1353911192689263,0.2657678059270839,0.92018779342723,1.4244455772241778,213 -Linear,Logistic,observational,False,6,0.95,0.9593114241001566,1.3529019638410478,0.2657678059270839,0.9624413145539906,1.6176444941500485,213 -Linear,Logistic,observational,True,1,0.9,0.8826291079812206,0.2823311571824021,0.06993758010197051,0.9295774647887324,0.3628548286921628,213 -Linear,Logistic,observational,True,1,0.95,0.9561815336463223,0.33641832362713375,0.06993758010197051,0.9577464788732394,0.40938919268144514,213 -Linear,Logistic,observational,True,4,0.9,0.4194053208137715,1.3815154955504438,0.876582736122305,0.2112676056338028,1.7315155396141317,213 -Linear,Logistic,observational,True,4,0.95,0.5273865414710485,1.6461772470181997,0.876582736122305,0.3145539906103286,1.9679400718614783,213 -Linear,Logistic,observational,True,6,0.9,0.9076682316118936,1.1396624657364007,0.2717053464179361,0.9154929577464789,1.430838330810308,213 -Linear,Logistic,observational,True,6,0.95,0.9608763693270735,1.3579915870783799,0.2717053464179361,0.9577464788732394,1.6241526383361697,213 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.37966666666666665,0.7094842843054401,0.5063966456271116,0.07,0.8833695236671398,1000 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.463,0.8454026680860791,0.5063966456271116,0.119,1.0075828733731322,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.48833333333333334,0.6093983296457081,0.3746967239327951,0.185,0.7748043075861907,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.5676666666666667,0.7261428973215824,0.3746967239327951,0.257,0.8788079154824321,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.8953333333333334,0.6039293782365326,0.15156143782863715,0.905,0.7671387444581327,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9506666666666667,0.7196262397785951,0.15156143782863715,0.952,0.8709721345495236,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.3783333333333333,0.7096389669594401,0.5063175240399332,0.066,0.8838830720898343,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.4603333333333333,0.8455869838366757,0.5063175240399332,0.124,1.0089102464690358,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.48533333333333334,0.609259041148495,0.37470955013293855,0.19,0.7754703791594748,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.5703333333333332,0.7259769248401865,0.37470955013293855,0.27,0.8784156068436955,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.8876666666666666,0.6038826619307712,0.15256760709225262,0.904,0.7681471625685046,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.948,0.7195705738668794,0.15256760709225262,0.954,0.8695993470187038,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.915,2.6608678733015485,0.688154090500346,0.931,3.366696329353361,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9656666666666667,3.1706196307305743,0.688154090500346,0.97,3.8192561069995286,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.903,3.556747699127985,0.9996416099478588,0.942,4.472327607652934,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.969,4.238126285623724,0.9996416099478588,0.984,5.088003820636524,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9373333333333334,2.1290784823926696,0.46227527065430596,0.956,2.703189636335836,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9743333333333334,2.5369534877597597,0.46227527065430596,0.991,3.0641146657072404,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.91,1.0977119557447321,0.2651109928804438,0.939,1.3939185973124866,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.968,1.3080044712830703,0.2651109928804438,0.981,1.580610122013623,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9316666666666666,1.4368875613066505,0.30987690101867293,0.952,1.8152404100048158,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.975,1.7121571329201994,0.30987690101867293,0.98,2.059654079760943,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.908,1.0154733246915484,0.23971463199529405,0.929,1.2938281773467855,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.96,1.210011098279508,0.23971463199529405,0.972,1.4652849487780493,1000 +Linear,Logistic,experimental,False,1,0.9,0.8176666666666667,0.2639777067863099,0.07735852107118578,0.764,0.3393389889591518,1000 +Linear,Logistic,experimental,False,1,0.95,0.891,0.3145488385988198,0.07735852107118578,0.839,0.3826999999595859,1000 +Linear,Logistic,experimental,False,4,0.9,0.31,1.0799528007716814,0.9173788544387992,0.045,1.361708244102074,1000 +Linear,Logistic,experimental,False,4,0.95,0.385,1.2868431329288754,0.9173788544387992,0.073,1.5474903461640912,1000 +Linear,Logistic,experimental,False,6,0.9,0.8963333333333334,1.0864824219772902,0.2703183844096557,0.893,1.3666259596934611,1000 +Linear,Logistic,experimental,False,6,0.95,0.949,1.2946236564879237,0.2703183844096557,0.943,1.5549441788053244,1000 +Linear,Logistic,experimental,True,1,0.9,0.8163333333333334,0.2639754882372008,0.07738204276505496,0.765,0.339331857870017,1000 +Linear,Logistic,experimental,True,1,0.95,0.8906666666666666,0.3145461950345047,0.07738204276505496,0.844,0.3833030006646118,1000 +Linear,Logistic,experimental,True,4,0.9,0.3103333333333333,1.0798559231989837,0.9168941023520879,0.046,1.3607552807065624,1000 +Linear,Logistic,experimental,True,4,0.95,0.3873333333333333,1.2867276961810175,0.9168941023520879,0.074,1.5475927725689025,1000 +Linear,Logistic,experimental,True,6,0.9,0.8986666666666666,1.0865640980568139,0.27054317418995893,0.896,1.3676856739628556,1000 +Linear,Logistic,experimental,True,6,0.95,0.95,1.2947209795394352,0.27054317418995893,0.941,1.5556519546992444,1000 +Linear,Logistic,observational,False,1,0.9,0.8956666666666666,0.28367316450513197,0.0700767685399924,0.903,0.3642537051104781,1000 +Linear,Logistic,observational,False,1,0.95,0.9523333333333334,0.3380174239825948,0.0700767685399924,0.948,0.4113211504413545,1000 +Linear,Logistic,observational,False,4,0.9,0.408,1.3776824648934105,0.9052340667640746,0.183,1.7236892476868426,1000 +Linear,Logistic,observational,False,4,0.95,0.5046666666666666,1.641609909282898,0.9052340667640746,0.277,1.964267689737536,1000 +Linear,Logistic,observational,False,6,0.9,0.8903333333333334,1.1309252917212975,0.282843171368907,0.89,1.4213783682706032,1000 +Linear,Logistic,observational,False,6,0.95,0.9493333333333334,1.3475806021033823,0.282843171368907,0.947,1.617450746310541,1000 +Linear,Logistic,observational,True,1,0.9,0.8876666666666666,0.28210363817731876,0.0702387016495634,0.9,0.36227342165859067,1000 +Linear,Logistic,observational,True,1,0.95,0.9513333333333334,0.3361472180111355,0.0702387016495634,0.945,0.40929779552981904,1000 +Linear,Logistic,observational,True,4,0.9,0.4043333333333333,1.3755156336478902,0.904918936330756,0.185,1.7226484388527301,1000 +Linear,Logistic,observational,True,4,0.95,0.497,1.6390279706032433,0.904918936330756,0.286,1.9605550323619565,1000 +Linear,Logistic,observational,True,6,0.9,0.892,1.125633507574361,0.2835718025458131,0.899,1.414839244504656,1000 +Linear,Logistic,observational,True,6,0.95,0.9473333333333334,1.341275052374208,0.2835718025458131,0.949,1.6097715181154,1000 diff --git a/results/did/did_multi_metadata.csv b/results/did/did_multi_metadata.csv index 67772b0..86381a7 100644 --- a/results/did/did_multi_metadata.csv +++ b/results/did/did_multi_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.11.dev0,DIDMultiCoverageSimulation,2025-06-02 17:51,339.9238995909691,3.12.3,scripts/did/did_pa_multi_config.yml +0.10.0,DIDMultiCoverageSimulation,2025-06-03 09:09,162.553562772274,3.12.9,scripts/did/did_pa_multi_config.yml diff --git a/results/did/did_multi_time.csv b/results/did/did_multi_time.csv index 8a5c303..3dda54d 100644 --- a/results/did/did_multi_time.csv +++ b/results/did/did_multi_time.csv @@ -1,49 +1,49 @@ Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.10485133020344288,0.6641311784095322,0.5823494355664303,0.08450704225352113,0.7881878804378276,213 -LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.17683881064162754,0.7913611091980961,0.5823494355664303,0.12206572769953052,0.9120924244627483,213 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.2222222222222222,0.5457603564040948,0.40420421511646165,0.15023474178403756,0.6619778117977112,213 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.34585289514866974,0.6503135751503121,0.40420421511646165,0.23943661971830985,0.7592098705244504,213 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.9107981220657277,0.5381868903349744,0.12772384273949355,0.9014084507042254,0.6551761588151264,213 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9530516431924883,0.6412892337193222,0.12772384273949355,0.9389671361502347,0.7509412548475333,213 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.10328638497652583,0.6638809210012889,0.5795783394754952,0.09389671361502347,0.7886190860615776,213 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.17527386541471046,0.7910629091035805,0.5795783394754952,0.1267605633802817,0.9100402277563315,213 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.2363067292644757,0.5454472863046809,0.40351084017097694,0.15492957746478872,0.6618382122506522,213 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.34115805946791866,0.6499405291178667,0.40351084017097694,0.2347417840375587,0.760586865832683,213 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.9045383411580594,0.5384698205489944,0.12841236979508242,0.9248826291079812,0.6548573031006936,213 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9577464788732394,0.6416263658631989,0.12841236979508242,0.9530516431924883,0.7496096842588228,213 -LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.917057902973396,2.826267846555917,0.6997630370632157,0.9342723004694836,3.505164210797084,213 -LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9624413145539906,3.367705854884923,0.6997630370632157,0.9765258215962441,4.010090570743562,213 -LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9061032863849765,3.9292910307842495,1.1032615377470605,0.9248826291079812,4.801480559342949,213 -LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9640062597809077,4.682039045253358,1.1032615377470605,0.9765258215962441,5.512445467612302,213 -LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9405320813771518,2.020686142877495,0.4287262814527601,0.9577464788732394,2.5196056115761056,213 -LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9765258215962441,2.4077960489647223,0.4287262814527601,0.9812206572769953,2.8786465586804595,213 -LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9420970266040688,1.0767197139991078,0.25231124461698884,0.9295774647887324,1.337727334792679,213 -LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9671361502347418,1.2829906724246045,0.25231124461698884,0.9671361502347418,1.521525908754415,213 -LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9389671361502347,1.4992929674517166,0.3404636513294964,0.9530516431924883,1.8343377037938378,213 -LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9780907668231612,1.7865177608087823,0.3404636513294964,0.9859154929577465,2.10956901976702,213 -LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9139280125195618,0.9226791828166745,0.22256304127174542,0.9248826291079812,1.1499561650632644,213 -LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9624413145539906,1.0994400583577786,0.22256304127174542,0.9765258215962441,1.3126410298530595,213 -Linear,Logistic,experimental,False,1,0.9,0.7981220657276995,0.24406856913925595,0.07455391600852629,0.7276995305164319,0.31257176314972324,213 -Linear,Logistic,experimental,False,1,0.95,0.863849765258216,0.29082563787621385,0.07455391600852629,0.8169014084507042,0.3530474954026036,213 -Linear,Logistic,experimental,False,4,0.9,0.046948356807511735,0.9683919238870853,1.0545074196771533,0.04225352112676056,1.1061614805926192,213 -Linear,Logistic,experimental,False,4,0.95,0.06572769953051644,1.153910148987462,1.0545074196771533,0.06103286384976526,1.2911717925367656,213 -Linear,Logistic,experimental,False,6,0.9,0.8982785602503913,0.9628742659936109,0.23137852998349606,0.892018779342723,1.1054779631893865,213 -Linear,Logistic,experimental,False,6,0.95,0.9452269170579031,1.147335454088764,0.23137852998349606,0.9624413145539906,1.289087298578374,213 -Linear,Logistic,experimental,True,1,0.9,0.8059467918622848,0.24408667991439068,0.07429939700420787,0.7370892018779343,0.3135963329097478,213 -Linear,Logistic,experimental,True,1,0.95,0.86697965571205,0.29084721819583287,0.07429939700420787,0.8262910798122066,0.35308221324877676,213 -Linear,Logistic,experimental,True,4,0.9,0.046948356807511735,0.9682481796882342,1.054760155003612,0.03286384976525822,1.1081060346576734,213 -Linear,Logistic,experimental,True,4,0.95,0.06572769953051644,1.1537388672100939,1.054760155003612,0.056338028169014086,1.2880440884321052,213 -Linear,Logistic,experimental,True,6,0.9,0.892018779342723,0.9628043535115562,0.23252849991729144,0.8873239436619719,1.1104069795500415,213 -Linear,Logistic,experimental,True,6,0.95,0.9499217527386542,1.1472521482281988,0.23252849991729144,0.9577464788732394,1.287530785006439,213 -Linear,Logistic,observational,False,1,0.9,0.9014084507042254,0.27469908871557547,0.06651861748549698,0.9014084507042254,0.3520501442016315,213 -Linear,Logistic,observational,False,1,0.95,0.9546165884194053,0.3273241531323111,0.06651861748549698,0.9436619718309859,0.39797250176020077,213 -Linear,Logistic,observational,False,4,0.9,0.18935837245696402,1.353982900128844,1.038997154582461,0.18309859154929578,1.5217737073595556,213 -Linear,Logistic,observational,False,4,0.95,0.27543035993740217,1.6133701360734638,1.038997154582461,0.26291079812206575,1.7800142309978275,213 -Linear,Logistic,observational,False,6,0.9,0.9045383411580594,1.0143473501307676,0.24543300102476093,0.9107981220657277,1.1667504579830554,213 -Linear,Logistic,observational,False,6,0.95,0.9593114241001566,1.208669416837173,0.24543300102476093,0.9577464788732394,1.3535074358284087,213 -Linear,Logistic,observational,True,1,0.9,0.8935837245696401,0.2725905663392203,0.06651898118558668,0.892018779342723,0.3498352468431658,213 -Linear,Logistic,observational,True,1,0.95,0.9530516431924883,0.32481169375566,0.06651898118558668,0.9436619718309859,0.3938902840125929,213 -Linear,Logistic,observational,True,4,0.9,0.19092331768388104,1.3586994912357533,1.029554721826275,0.1784037558685446,1.5283855406123952,213 -Linear,Logistic,observational,True,4,0.95,0.2863849765258216,1.6189903010218047,1.029554721826275,0.26291079812206575,1.7818990446425318,213 -Linear,Logistic,observational,True,6,0.9,0.895148669796557,1.0131038455549137,0.24852464288526532,0.8967136150234741,1.1643867720039867,213 -Linear,Logistic,observational,True,6,0.95,0.9530516431924883,1.2071876897440446,0.24852464288526532,0.9530516431924883,1.3557343370847632,213 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.09633333333333333,0.6730703224735359,0.5813847440636578,0.06,0.7994113182366382,1000 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.163,0.8020127563300868,0.5813847440636578,0.109,0.9222994395169547,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.20866666666666667,0.545809228523173,0.4318166485465945,0.148,0.6621126753160614,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.29,0.6503718098720356,0.4318166485465945,0.226,0.7607781957690535,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.8883333333333334,0.539451744339498,0.1374713900065768,0.888,0.6559678253242532,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.943,0.6427964002257806,0.1374713900065768,0.942,0.7532136346784374,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.095,0.673215344660322,0.5816575606523053,0.066,0.7987689454841812,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.16966666666666666,0.8021855609240048,0.5816575606523053,0.116,0.9211448037237728,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.212,0.5457365545641853,0.43118597517736307,0.158,0.662287763310214,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.29633333333333334,0.6502852135087509,0.43118597517736307,0.223,0.7601307241685876,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.882,0.5395052157780649,0.13917064701840512,0.889,0.6561053221874785,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.938,0.6428601153747003,0.13917064701840512,0.947,0.753120590966105,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.8976666666666666,2.8641489580508317,0.7473265850366615,0.92,3.5454942167899373,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.962,3.41284398329212,0.7473265850366615,0.97,4.046408625403042,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.8876666666666666,3.9648529818754157,1.1487073320114483,0.918,4.845766161863059,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9593333333333334,4.724413723593485,1.1487073320114483,0.986,5.548216313166135,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.938,2.0148113447449116,0.43840143925357783,0.955,2.5144593978114727,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9783333333333334,2.4007957952232104,0.43840143925357783,0.99,2.861410408432865,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9076666666666666,1.1003301479666492,0.27099358031240817,0.913,1.3627159220412297,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9583333333333334,1.311124239738742,0.27099358031240817,0.97,1.5572874751641161,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9333333333333333,1.4854173027590014,0.3229586888066522,0.95,1.8150130378533484,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.976,1.7699838865395685,0.3229586888066522,0.978,2.0803451880507104,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9163333333333333,0.9353518863595044,0.219408785264352,0.912,1.1653988211882675,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9603333333333334,1.1145405160056325,0.219408785264352,0.959,1.3283422144863115,1000 +Linear,Logistic,experimental,False,1,0.9,0.7966666666666666,0.24428496421942136,0.07478155567895448,0.728,0.3129228235483047,1000 +Linear,Logistic,experimental,False,1,0.95,0.8733333333333334,0.2910834885181229,0.07478155567895448,0.833,0.35375833173558197,1000 +Linear,Logistic,experimental,False,4,0.9,0.038,0.9678137745111298,1.0814213490525186,0.03,1.1087428056263964,1000 +Linear,Logistic,experimental,False,4,0.95,0.06666666666666667,1.1532212415150949,1.0814213490525186,0.054,1.2896096218626036,1000 +Linear,Logistic,experimental,False,6,0.9,0.8883333333333334,0.9645120733919993,0.2398038156236055,0.884,1.1089972839130895,1000 +Linear,Logistic,experimental,False,6,0.95,0.9386666666666666,1.1492870219741105,0.2398038156236055,0.939,1.2860092104213576,1000 +Linear,Logistic,experimental,True,1,0.9,0.7976666666666666,0.244274323547519,0.0748075511921299,0.735,0.31288102428256387,1000 +Linear,Logistic,experimental,True,1,0.95,0.8716666666666666,0.2910708093755182,0.0748075511921299,0.833,0.35371101065661276,1000 +Linear,Logistic,experimental,True,4,0.9,0.03833333333333333,0.9676295529475489,1.0812355403537053,0.03,1.10872071383113,1000 +Linear,Logistic,experimental,True,4,0.95,0.067,1.1530017279827793,1.0812355403537053,0.054,1.289018545293176,1000 +Linear,Logistic,experimental,True,6,0.9,0.885,0.964486553866753,0.23961465238247193,0.887,1.109074934480792,1000 +Linear,Logistic,experimental,True,6,0.95,0.9383333333333334,1.1492566135842297,0.23961465238247193,0.933,1.2873487345102677,1000 +Linear,Logistic,observational,False,1,0.9,0.8873333333333334,0.2738660718613766,0.0672792367714224,0.875,0.35107388214992413,1000 +Linear,Logistic,observational,False,1,0.95,0.9376666666666666,0.32633155232820765,0.0672792367714224,0.938,0.39679498726255896,1000 +Linear,Logistic,observational,False,4,0.9,0.16433333333333333,1.3471569564681332,1.0662036471383771,0.133,1.5189433786007114,1000 +Linear,Logistic,observational,False,4,0.95,0.248,1.6052365225310308,1.0662036471383771,0.203,1.7701614407109725,1000 +Linear,Logistic,observational,False,6,0.9,0.8823333333333334,1.0106046290695898,0.25297949046206,0.886,1.1632254064709615,1000 +Linear,Logistic,observational,False,6,0.95,0.9396666666666667,1.20420968962261,0.25297949046206,0.941,1.3507861334703333,1000 +Linear,Logistic,observational,True,1,0.9,0.8836666666666666,0.27183527948043773,0.06710802675280161,0.87,0.3482504195565255,1000 +Linear,Logistic,observational,True,1,0.95,0.9373333333333334,0.3239117139538379,0.06710802675280161,0.934,0.3934241047155358,1000 +Linear,Logistic,observational,True,4,0.9,0.17166666666666666,1.3476603584428708,1.0647967745035234,0.133,1.5187476579231214,1000 +Linear,Logistic,observational,True,4,0.95,0.247,1.6058363629813088,1.0647967745035234,0.207,1.7714708628832339,1000 +Linear,Logistic,observational,True,6,0.9,0.889,1.0055746633220144,0.25465275272035065,0.884,1.1560383777095113,1000 +Linear,Logistic,observational,True,6,0.95,0.9386666666666666,1.1982161157585394,0.25465275272035065,0.939,1.3436875509579689,1000