diff --git a/.github/workflows/did_sim.yml b/.github/workflows/did_sim.yml index 199220d..411502f 100644 --- a/.github/workflows/did_sim.yml +++ b/.github/workflows/did_sim.yml @@ -52,7 +52,7 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v5 with: - version: "0.6.11" + version: "0.7.8" - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/rdd_sim.yml b/.github/workflows/rdd_sim.yml index e79bee9..ea490ee 100644 --- a/.github/workflows/rdd_sim.yml +++ b/.github/workflows/rdd_sim.yml @@ -17,8 +17,8 @@ jobs: strategy: matrix: script: [ - 'scripts/rdd/rdd_sharp_coverage.py', - 'scripts/rdd/rdd_fuzzy_coverage.py', + 'scripts/rdd/rdd_sharp.py', + 'scripts/rdd/rdd_fuzzy.py', ] steps: @@ -48,26 +48,32 @@ jobs: with: ref: ${{ env.TARGET_BRANCH }} + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + version: "0.7.8" + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version-file: "monte-cover/pyproject.toml" - - name: Install dependencies + - name: Install Monte-Cover run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + cd monte-cover + uv venv + uv sync - - name: Install DoubleML from correct branch - run: | - pip uninstall -y doubleml - pip install "doubleml[rdd] @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' - - name: Install RDFlex from main branch + - name: Install DoubleML from correct branch run: | - pip uninstall -y doubleml - pip install git+https://github.com/DoubleML/doubleml-rdflex.git@main - pip install rdrobust + source monte-cover/.venv/bin/activate + uv pip uninstall doubleml + uv pip install "doubleml[rdd] @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}" - name: Set up Git configuration run: | @@ -75,7 +81,9 @@ jobs: git config --global user.email 'github-actions@github.com' - name: Run scripts - run: python ${{ matrix.script }} + run: | + source monte-cover/.venv/bin/activate + uv run ${{ matrix.script }} - name: Commit any existing changes run: | diff --git a/doc/rdd/rdd.qmd b/doc/rdd/rdd.qmd index 4a74fb7..dce518d 100644 --- a/doc/rdd/rdd.qmd +++ b/doc/rdd/rdd.qmd @@ -47,7 +47,7 @@ df_sharp = pd.read_csv("../../results/rdd/rdd_sharp_coverage.csv", index_col=Non assert df_sharp["repetition"].nunique() == 1 n_rep_sharp = df_sharp["repetition"].unique()[0] -display_columns_sharp = ["Method", "Learner g", "fs specification", "Bias", "CI Length", "Coverage"] +display_columns_sharp = ["Method", "Learner g", "fs_specification", "Bias", "CI Length", "Coverage"] ``` ```{python} @@ -99,7 +99,7 @@ df_fuzzy = pd.read_csv("../../results/rdd/rdd_fuzzy_coverage.csv", index_col=Non assert df_fuzzy["repetition"].nunique() == 1 n_rep_fuzzy = df_fuzzy["repetition"].unique()[0] -display_columns_fuzzy = ["Method", "Learner g", "Learner m", "fs specification", "Bias", "CI Length", "Coverage"] +display_columns_fuzzy = ["Method", "Learner g", "Learner m", "fs_specification", "Bias", "CI Length", "Coverage"] ``` ```{python} diff --git a/monte-cover/src/montecover/did/did_pa_multi.py b/monte-cover/src/montecover/did/did_pa_multi.py index fa6aeb6..eb84934 100644 --- a/monte-cover/src/montecover/did/did_pa_multi.py +++ b/monte-cover/src/montecover/did/did_pa_multi.py @@ -4,10 +4,9 @@ import numpy as np import pandas as pd from doubleml.did.datasets import make_did_CS2021 -from lightgbm import LGBMClassifier, LGBMRegressor -from sklearn.linear_model import LinearRegression, LogisticRegression from montecover.base import BaseSimulation +from montecover.utils import create_learner_from_config class DIDMultiCoverageSimulation(BaseSimulation): @@ -36,39 +35,13 @@ def __init__( def _process_config_parameters(self): """Process simulation-specific parameters from config""" # Process ML models in parameter grid + # Process ML models in parameter grid + assert "learners" in self.dml_parameters, "No learners specified in the config file" - assert ( - "learners" in self.dml_parameters - ), "No learners specified in the config file" + required_learners = ["ml_g", "ml_m"] for learner in self.dml_parameters["learners"]: - assert "ml_g" in learner, "No ml_g specified in the config file" - assert "ml_m" in learner, "No ml_m specified in the config file" - - # Convert ml_g strings to actual objects - if learner["ml_g"][0] == "Linear": - learner["ml_g"] = ("Linear", LinearRegression()) - elif learner["ml_g"][0] == "LGBM": - learner["ml_g"] = ( - "LGBM", - LGBMRegressor( - n_estimators=500, learning_rate=0.02, verbose=-1, n_jobs=1 - ), - ) - else: - raise ValueError(f"Unknown learner type: {learner['ml_g']}") - - # Convert ml_m strings to actual objects - if learner["ml_m"][0] == "Linear": - learner["ml_m"] = ("Linear", LogisticRegression()) - elif learner["ml_m"][0] == "LGBM": - learner["ml_m"] = ( - "LGBM", - LGBMClassifier( - n_estimators=500, learning_rate=0.02, verbose=-1, n_jobs=1 - ), - ) - else: - raise ValueError(f"Unknown learner type: {learner['ml_m']}") + for ml in required_learners: + assert ml in learner, f"No {ml} specified in the config file" def _calculate_oracle_values(self): """Calculate oracle values for the simulation.""" @@ -102,8 +75,9 @@ def _calculate_oracle_values(self): def run_single_rep(self, dml_data, dml_params) -> Dict[str, Any]: """Run a single repetition with the given parameters.""" # Extract parameters - learner_g_name, ml_g = dml_params["learners"]["ml_g"] - learner_m_name, ml_m = dml_params["learners"]["ml_m"] + learner_config = dml_params["learners"] + learner_g_name, ml_g = create_learner_from_config(learner_config["ml_g"]) + learner_m_name, ml_m = create_learner_from_config(learner_config["ml_m"]) score = dml_params["score"] in_sample_normalization = dml_params["in_sample_normalization"] diff --git a/monte-cover/src/montecover/rdd/__init__.py b/monte-cover/src/montecover/rdd/__init__.py new file mode 100644 index 0000000..50efadb --- /dev/null +++ b/monte-cover/src/montecover/rdd/__init__.py @@ -0,0 +1,7 @@ +"""Monte Carlo coverage simulations for RDD.""" + +from montecover.rdd.rdd import RDDCoverageSimulation + +__all__ = [ + "RDDCoverageSimulation", +] diff --git a/monte-cover/src/montecover/rdd/rdd.py b/monte-cover/src/montecover/rdd/rdd.py new file mode 100644 index 0000000..8c36d80 --- /dev/null +++ b/monte-cover/src/montecover/rdd/rdd.py @@ -0,0 +1,239 @@ +import time +import warnings +from itertools import product +from typing import Any, Dict, Optional + +import doubleml as dml +import numpy as np +import pandas as pd +from doubleml.rdd.datasets import make_simple_rdd_data +from rdrobust import rdrobust +from statsmodels.nonparametric.kernel_regression import KernelReg + +from montecover.base import BaseSimulation +from montecover.utils import create_learner_from_config + + +class RDDCoverageSimulation(BaseSimulation): + """Simulation class for coverage properties of DoubleML RDFlex for RDD.""" + + def __init__( + self, + config_file: str, + suppress_warnings: bool = True, + log_level: str = "INFO", + log_file: Optional[str] = None, + ): + super().__init__( + config_file=config_file, + suppress_warnings=suppress_warnings, + log_level=log_level, + log_file=log_file, + ) + + self.fuzzy = self.dgp_parameters.get("fuzzy", [False])[0] + self.cutoff = self.dgp_parameters.get("cutoff", [0.0])[0] + # Calculate oracle values + self._calculate_oracle_values() + + def _process_config_parameters(self): + """Process simulation-specific parameters from config.""" + + # Process ML models in parameter grid + assert "learners" in self.dml_parameters, "No learners specified in the config file" + + required_learners = ["ml_g"] + for learner in self.dml_parameters["learners"]: + for ml in required_learners: + assert ml in learner, f"No {ml} specified in the config file" + + def _calculate_oracle_values(self): + """Calculate oracle values for the simulation.""" + self.logger.info("Calculating oracle values") + + data_oracle = make_simple_rdd_data(n_obs=int(1e6), fuzzy=self.fuzzy, cutoff=self.cutoff) + # get oracle value + score = data_oracle["score"] + ite = data_oracle["oracle_values"]["Y1"] - data_oracle["oracle_values"]["Y0"] + + # subset score and ite for faster computation + score_subset = (score >= (self.cutoff - 0.02)) & (score <= (self.cutoff + 0.02)) + self.logger.info(f"Oracle score subset size: {np.sum(score_subset)}") + kernel_reg = KernelReg(endog=ite[score_subset], exog=score[score_subset], var_type="c", reg_type="ll") + effect_at_cutoff, _ = kernel_reg.fit(np.array([self.cutoff])) + oracle_effect = effect_at_cutoff[0] + + self.logger.info(f"Oracle effect at cutoff: {oracle_effect}") + self.oracle_values = dict() + self.oracle_values["theta"] = oracle_effect + + def _process_repetition(self, i_rep): + """Process a single repetition with all parameter combinations.""" + if self.suppress_warnings: + warnings.simplefilter(action="ignore", category=UserWarning) + + i_param_comb = 0 + rep_results = { + "coverage": [], + } + + # loop through all parameter combinations + for dgp_param_values in product(*self.dgp_parameters.values()): + dgp_params = dict(zip(self.dgp_parameters.keys(), dgp_param_values)) + dml_data = self._generate_dml_data(dgp_params) + + # --- Run rdrobust benchmark --- + self.logger.debug(f"Rep {i_rep+1}: Running rdrobust benchmark for DGP {dgp_params}") + param_start_time_rd_benchmark = time.time() + + # Call the dedicated benchmark function + # Pass dml_data, current dgp_params, and repetition index + benchmark_result_list = self._rdrobust_benchmark(dml_data, dgp_params, i_rep) + if benchmark_result_list: + rep_results["coverage"].extend(benchmark_result_list) + + param_duration_rd_benchmark = time.time() - param_start_time_rd_benchmark + self.logger.debug(f"rdrobust benchmark for DGP {dgp_params} completed in {param_duration_rd_benchmark:.2f}s") + + for dml_param_values in product(*self.dml_parameters.values()): + dml_params = dict(zip(self.dml_parameters.keys(), dml_param_values)) + i_param_comb += 1 + + comb_results = self._process_parameter_combination(i_rep, i_param_comb, dgp_params, dml_params, dml_data) + rep_results["coverage"].extend(comb_results["coverage"]) + + return rep_results + + def _rdrobust_benchmark(self, dml_data, dml_params, i_rep): + """Run a benchmark using rdrobust for RDD.""" + + # Extract parameters + score = dml_data.data[dml_data.s_col] + Y = dml_data.data[dml_data.y_col] + Z = dml_data.data[dml_data.x_cols] + + benchmark_results_list = [] + for level in self.confidence_parameters["level"]: + if self.fuzzy: + D = dml_data.data[dml_data.d_cols] + rd_model = rdrobust(y=Y, x=score, fuzzy=D, covs=Z, c=self.cutoff, level=level * 100) + else: + rd_model = rdrobust(y=Y, x=score, covs=Z, c=self.cutoff, level=level * 100) + coef_rd = rd_model.coef.loc["Robust", "Coeff"] + ci_lower_rd = rd_model.ci.loc["Robust", "CI Lower"] + ci_upper_rd = rd_model.ci.loc["Robust", "CI Upper"] + + confint_for_compute = pd.DataFrame({"lower": [ci_lower_rd], "upper": [ci_upper_rd]}) + theta_for_compute = np.array([coef_rd]) + + coverage_metrics = self._compute_coverage( + thetas=theta_for_compute, + oracle_thetas=self.oracle_values["theta"], + confint=confint_for_compute, + joint_confint=None, + ) + + # Add metadata + coverage_metrics.update( + { + "repetition": i_rep, + "Learner g": "Linear", + "Learner m": "Logistic", + "Method": "rdrobust", + "fs_specification": "cutoff", + "level": level, + } + ) + benchmark_results_list.append(coverage_metrics) + + return benchmark_results_list + + def run_single_rep(self, dml_data, dml_params) -> Dict[str, Any]: + """Run a single repetition with the given parameters.""" + + # Extract parameters + learner_config = dml_params["learners"] + learner_g_name, ml_g = create_learner_from_config(learner_config["ml_g"]) + if self.fuzzy: + learner_m_name, ml_m = create_learner_from_config(learner_config["ml_m"]) + else: + learner_m_name, ml_m = "N/A", None + fs_specification = dml_params["fs_specification"] + + # Model + dml_model = dml.rdd.RDFlex( + obj_dml_data=dml_data, + ml_g=ml_g, + ml_m=ml_m, + n_folds=5, + n_rep=1, + fuzzy=self.fuzzy, + cutoff=self.cutoff, + fs_specification=fs_specification, + ) + dml_model.fit() + + result = { + "coverage": [], + } + for level in self.confidence_parameters["level"]: + level_result = dict() + level_result["coverage"] = self._compute_coverage( + thetas=dml_model.coef, + oracle_thetas=self.oracle_values["theta"], + confint=dml_model.confint(level=level), + joint_confint=None, + ) + + # add parameters to the result + for res in level_result.values(): + res.update( + { + "Learner g": learner_g_name, + "Learner m": learner_m_name, + "Method": "RDFlex", + "fs_specification": fs_specification, + "level": level, + } + ) + for key, res in level_result.items(): + result[key].append(res) + + return result + + def summarize_results(self): + """Summarize the simulation results.""" + self.logger.info("Summarizing simulation results") + + # Group by parameter combinations + groupby_cols = ["Method", "fs_specification", "Learner g", "Learner m", "level"] + aggregation_dict = { + "Coverage": "mean", + "CI Length": "mean", + "Bias": "mean", + "repetition": "count", + } + + # Aggregate results (possibly multiple result dfs) + result_summary = dict() + for result_name, result_df in self.results.items(): + result_summary[result_name] = result_df.groupby(groupby_cols).agg(aggregation_dict).reset_index() + self.logger.debug(f"Summarized {result_name} results") + + return result_summary + + def _generate_dml_data(self, dgp_params) -> dml.DoubleMLData: + """Generate data for the simulation.""" + data = make_simple_rdd_data( + n_obs=dgp_params["n_obs"], + fuzzy=dgp_params["fuzzy"], + cutoff=dgp_params["cutoff"], + ) + + score = data["score"] + Y = data["Y"] + X = data["X"].reshape(dgp_params["n_obs"], -1) + D = data["D"] + + dml_data = dml.DoubleMLData.from_arrays(y=Y, d=D, x=X, s=score) + return dml_data diff --git a/monte-cover/src/montecover/utils.py b/monte-cover/src/montecover/utils.py index b73dee4..838cb43 100644 --- a/monte-cover/src/montecover/utils.py +++ b/monte-cover/src/montecover/utils.py @@ -1,8 +1,9 @@ from typing import Any, Callable, Dict, Tuple +from doubleml.utils import GlobalClassifier, GlobalRegressor from lightgbm import LGBMClassifier, LGBMRegressor -from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor -from sklearn.linear_model import LassoCV, LinearRegression, LogisticRegression +from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor, StackingClassifier, StackingRegressor +from sklearn.linear_model import LassoCV, LinearRegression, LogisticRegression, Ridge LearnerInstantiator = Callable[[Dict[str, Any]], Any] # Map learner abbreviations to their instantiation logic @@ -14,6 +15,30 @@ "LGBM Clas.": lambda params: LGBMClassifier(**{**{"verbose": -1, "n_jobs": 1}, **params}), "Linear": lambda params: LinearRegression(**params), "Logistic": lambda params: LogisticRegression(**params), + "Global Linear": lambda params: GlobalRegressor(LinearRegression(**params)), + "Global Logistic": lambda params: GlobalClassifier(LogisticRegression(**params)), + "Stacked Regr.": lambda params: StackingRegressor( + estimators=[ + ("lr", LinearRegression()), + ( + "lgbm", + LGBMRegressor(**{**{"verbose": -1, "n_jobs": 1}, **params}), + ), + ("glr", GlobalRegressor(LinearRegression())), + ], + final_estimator=Ridge(), + ), + "Stacked Clas.": lambda params: StackingClassifier( + estimators=[ + ("lr", LogisticRegression()), + ( + "lgbm", + LGBMClassifier(**{**{"verbose": -1, "n_jobs": 1}, **params}), + ), + ("glr", GlobalClassifier(LogisticRegression())), + ], + final_estimator=LogisticRegression(), + ), } diff --git a/results/did/did_cs_atte_coverage_metadata.csv b/results/did/did_cs_atte_coverage_metadata.csv index 19aa007..c055b9e 100644 --- a/results/did/did_cs_atte_coverage_metadata.csv +++ b/results/did/did_cs_atte_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,did_cs_atte_coverage.py,2025-05-22 15:05:03,12767.107241630554,3.12.3 +0.11.dev0,did_cs_atte_coverage.py,2025-06-02 15:42:26,12669.156663179398,3.12.3 diff --git a/results/did/did_multi_detailed.csv b/results/did/did_multi_detailed.csv index 3384909..ab7ab8b 100644 --- a/results/did/did_multi_detailed.csv +++ b/results/did/did_multi_detailed.csv @@ -1,49 +1,49 @@ Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,experimental,False,1,0.9,0.4084166666666667,0.6711379447655711,0.45573140858217215,0.079,1.0050739084247675,1000 -LGBM,LGBM,experimental,False,1,0.95,0.49625,0.7997101862715227,0.45573140858217215,0.132,1.115181838446232,1000 -LGBM,LGBM,experimental,False,4,0.9,0.5408333333333334,0.5829183572257842,0.3256247127712507,0.227,0.8979506620328224,1000 -LGBM,LGBM,experimental,False,4,0.95,0.62975,0.6945900640455575,0.3256247127712507,0.299,0.9884673580918313,1000 -LGBM,LGBM,experimental,False,6,0.9,0.8979166666666666,0.5800302078090503,0.14243775358828042,0.89,0.892704703388473,1000 -LGBM,LGBM,experimental,False,6,0.95,0.95,0.691148621751838,0.14243775358828042,0.955,0.9829906161123636,1000 -LGBM,LGBM,experimental,True,1,0.9,0.4105833333333333,0.671233414997924,0.45509646764918477,0.081,1.005300983475438,1000 -LGBM,LGBM,experimental,True,1,0.95,0.4990833333333333,0.7998239460699273,0.45509646764918477,0.139,1.1154305467306445,1000 -LGBM,LGBM,experimental,True,4,0.9,0.5370833333333334,0.5828484393127565,0.3258110613161524,0.212,0.8973624125797166,1000 -LGBM,LGBM,experimental,True,4,0.95,0.6318333333333334,0.6945067517135889,0.3258110613161524,0.305,0.9879336259779586,1000 -LGBM,LGBM,experimental,True,6,0.9,0.89625,0.5799071420563836,0.14138962778278252,0.903,0.8926727198727438,1000 -LGBM,LGBM,experimental,True,6,0.95,0.947,0.6910019798628547,0.14138962778278252,0.954,0.982776272537425,1000 -LGBM,LGBM,observational,False,1,0.9,0.90725,2.7320711716537267,0.7081787206178205,0.946,4.2481549947682335,1000 -LGBM,LGBM,observational,False,1,0.95,0.9646666666666667,3.255463593782398,0.7081787206178205,0.985,4.664072261741128,1000 -LGBM,LGBM,observational,False,4,0.9,0.9076666666666666,3.5140296061122283,0.9722309202462336,0.972,5.406999616722633,1000 -LGBM,LGBM,observational,False,4,0.95,0.9648333333333333,4.187224538241929,0.9722309202462336,0.995,5.954805878504198,1000 -LGBM,LGBM,observational,False,6,0.9,0.92525,2.166594877755592,0.5140262158838963,0.96,3.3799046023620645,1000 -LGBM,LGBM,observational,False,6,0.95,0.9675833333333334,2.5816570300909847,0.5140262158838963,0.984,3.7082268657228123,1000 -LGBM,LGBM,observational,True,1,0.9,0.9088333333333334,1.1285770515711326,0.2779542066229801,0.934,1.7607428145840895,1000 -LGBM,LGBM,observational,True,1,0.95,0.9595833333333333,1.3447825013812504,0.2779542066229801,0.971,1.9310262593666496,1000 -LGBM,LGBM,observational,True,4,0.9,0.92175,1.4119321412730104,0.3268873239533469,0.941,2.1833548301330525,1000 -LGBM,LGBM,observational,True,4,0.95,0.9650833333333334,1.6824209158589551,0.3268873239533469,0.975,2.400470983600388,1000 -LGBM,LGBM,observational,True,6,0.9,0.9054166666666666,1.0205289656802177,0.2486678941122539,0.917,1.597156856017389,1000 -LGBM,LGBM,observational,True,6,0.95,0.9546666666666667,1.2160352660803362,0.2486678941122539,0.957,1.7513439251743073,1000 -Linear,Linear,experimental,False,1,0.9,0.84575,0.2947177158765365,0.08161611344312397,0.752,0.4590941708451621,1000 -Linear,Linear,experimental,False,1,0.95,0.9100833333333334,0.35117781865763664,0.08161611344312397,0.853,0.504099184053661,1000 -Linear,Linear,experimental,False,4,0.9,0.3073333333333333,0.974825708310707,0.808795845992618,0.033,1.4112500397019019,1000 -Linear,Linear,experimental,False,4,0.95,0.38408333333333333,1.1615764759772769,0.808795845992618,0.069,1.5731674507528597,1000 -Linear,Linear,experimental,False,6,0.9,0.8911666666666667,0.9832819402273247,0.243739180309483,0.893,1.4205928620859418,1000 -Linear,Linear,experimental,False,6,0.95,0.9423333333333334,1.171652697794173,0.243739180309483,0.949,1.5853446204128676,1000 -Linear,Linear,experimental,True,1,0.9,0.8463333333333334,0.2947197524013989,0.08159475357883039,0.759,0.4593380570357237,1000 -Linear,Linear,experimental,True,1,0.95,0.9099166666666666,0.351180245326684,0.08159475357883039,0.86,0.5045635273614648,1000 -Linear,Linear,experimental,True,4,0.9,0.30625,0.9748445034208943,0.8085550484761943,0.034,1.4110989059663714,1000 -Linear,Linear,experimental,True,4,0.95,0.38475,1.1615988717324064,0.8085550484761943,0.068,1.574069732455078,1000 -Linear,Linear,experimental,True,6,0.9,0.89,0.9832818903405501,0.24368928454372696,0.893,1.4193554640817743,1000 -Linear,Linear,experimental,True,6,0.95,0.94225,1.1716526383504147,0.24368928454372696,0.952,1.5837169582518265,1000 -Linear,Linear,observational,False,1,0.9,0.9005,0.3188204993145348,0.07710334427560887,0.894,0.495905282385802,1000 -Linear,Linear,observational,False,1,0.95,0.9494166666666666,0.37989805655090114,0.07710334427560887,0.948,0.5446250222430568,1000 -Linear,Linear,observational,False,4,0.9,0.42083333333333334,1.2366389453819784,0.7873737934686624,0.183,1.7673158966945293,1000 -Linear,Linear,observational,False,4,0.95,0.527,1.4735461898335716,0.7873737934686624,0.272,1.977244843861637,1000 -Linear,Linear,observational,False,6,0.9,0.8901666666666667,1.0315660555384851,0.2592484851241425,0.889,1.4876338955955155,1000 -Linear,Linear,observational,False,6,0.95,0.9431666666666666,1.2291867698140935,0.2592484851241425,0.937,1.6596741796186278,1000 -Linear,Linear,observational,True,1,0.9,0.8986666666666666,0.31665078693066356,0.07694111946321994,0.883,0.49251977204037817,1000 -Linear,Linear,observational,True,1,0.95,0.9495,0.37731268478315316,0.07694111946321994,0.944,0.5411752370665596,1000 -Linear,Linear,observational,True,4,0.9,0.417,1.2352689208194765,0.7872341789805914,0.183,1.7660298619152044,1000 -Linear,Linear,observational,True,4,0.95,0.5238333333333334,1.4719137048778035,0.7872341789805914,0.278,1.9751958823009785,1000 -Linear,Linear,observational,True,6,0.9,0.8873333333333334,1.023987089801951,0.259336442252307,0.88,1.4789961469734276,1000 -Linear,Linear,observational,True,6,0.95,0.9425,1.2201558751251838,0.259336442252307,0.947,1.6504862891485015,1000 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.4005833333333333,0.668417462452362,0.4514209317520677,0.069,1.001536224942987,1000 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.49541666666666667,0.7964685316542967,0.4514209317520677,0.119,1.1096223780406622,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.525,0.5836476641555988,0.3357039587486384,0.192,0.8984549944516823,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.6135,0.6954590868526271,0.3357039587486384,0.266,0.9889489428407441,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.8913333333333334,0.5798205422267835,0.1436401623087172,0.896,0.8921080477762784,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9469166666666666,0.6908987898012575,0.1436401623087172,0.95,0.9823087136174479,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.403,0.668486737121192,0.4516172220554326,0.064,1.0020517044274269,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.4925,0.7965510775135288,0.4516172220554326,0.118,1.110697612751064,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.5244166666666666,0.5834979623893575,0.33563072303718966,0.202,0.8980797806262651,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.6114166666666666,0.6952807061958639,0.33563072303718966,0.275,0.9886661012047989,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.8903333333333334,0.5798432460053057,0.1452858251598464,0.888,0.8930518639150258,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9449166666666666,0.690925843021974,0.1452858251598464,0.942,0.9832578157026843,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9115,2.7279550989921355,0.7081663387461643,0.951,4.239219420515267,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9635833333333333,3.250558990696578,0.7081663387461643,0.988,4.6533388042213675,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9074166666666666,3.5225937797559883,0.9757809691235549,0.97,5.426540882685481,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9669166666666666,4.19742938055986,0.9757809691235549,0.992,5.969933156329405,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9271666666666666,2.1769181957500305,0.5048068220768145,0.971,3.397149166606052,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9709166666666667,2.5939580221905385,0.5048068220768145,0.996,3.7281189530691132,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.90375,1.1192914856600868,0.27862527337620824,0.928,1.7467988922140234,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9589166666666666,1.3337180671583386,0.27862527337620824,0.967,1.914692647031146,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9199166666666666,1.4197310386522546,0.32827809352307774,0.935,2.19395277603094,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9664166666666666,1.6917138752639644,0.32827809352307774,0.968,2.4119391624873256,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9005,1.0294284373000027,0.2502841114417097,0.917,1.6092703629556147,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9533333333333334,1.226639640579326,0.2502841114417097,0.971,1.7641639749309233,1000 +Linear,Logistic,experimental,False,1,0.9,0.8521666666666666,0.2947207725579002,0.08125759297583215,0.769,0.45936907943411753,1000 +Linear,Logistic,experimental,False,1,0.95,0.91225,0.3511814609181314,0.08125759297583215,0.857,0.5043035205016457,1000 +Linear,Logistic,experimental,False,4,0.9,0.3186666666666667,0.9766434317991218,0.8093782242316307,0.041,1.4141166689592581,1000 +Linear,Logistic,experimental,False,4,0.95,0.393,1.1637424271067696,0.8093782242316307,0.079,1.5775150069804904,1000 +Linear,Logistic,experimental,False,6,0.9,0.8959166666666666,0.9840677614600047,0.24403762906388674,0.889,1.4217412744627138,1000 +Linear,Logistic,experimental,False,6,0.95,0.9463333333333334,1.1725890615466086,0.24403762906388674,0.94,1.585826837431638,1000 +Linear,Logistic,experimental,True,1,0.9,0.8528333333333333,0.29471605836940884,0.08132561217659459,0.764,0.45923649733243366,1000 +Linear,Logistic,experimental,True,1,0.95,0.9120833333333334,0.351175843616076,0.08132561217659459,0.855,0.5041154397102058,1000 +Linear,Logistic,experimental,True,4,0.9,0.3188333333333333,0.9765337199818319,0.8092231167974461,0.041,1.4131241013020541,1000 +Linear,Logistic,experimental,True,4,0.95,0.39458333333333334,1.1636116974132316,0.8092231167974461,0.076,1.5757045631534141,1000 +Linear,Logistic,experimental,True,6,0.9,0.8968333333333334,0.984151005519869,0.24420883449452355,0.889,1.4208221899399172,1000 +Linear,Logistic,experimental,True,6,0.95,0.94675,1.1726882529619342,0.24420883449452355,0.936,1.5847814464063683,1000 +Linear,Logistic,observational,False,1,0.9,0.9001666666666667,0.3180779030235929,0.0773596882018309,0.88,0.49501077303673774,1000 +Linear,Logistic,observational,False,1,0.95,0.94775,0.3790131984933507,0.0773596882018309,0.947,0.5435398448787098,1000 +Linear,Logistic,observational,False,4,0.9,0.4245,1.237493689045589,0.7914353096522312,0.18,1.76761883251908,1000 +Linear,Logistic,observational,False,4,0.95,0.5209166666666666,1.4745646797278944,0.7914353096522312,0.275,1.9755226904473104,1000 +Linear,Logistic,observational,False,6,0.9,0.8929166666666666,1.0255283640310429,0.255700122939389,0.895,1.4816116459637685,1000 +Linear,Logistic,observational,False,6,0.95,0.9455,1.221992416644637,0.255700122939389,0.934,1.653527148130022,1000 +Linear,Logistic,observational,True,1,0.9,0.8965,0.31619901580629955,0.07736324119662329,0.885,0.49203725267016324,1000 +Linear,Logistic,observational,True,1,0.95,0.9463333333333334,0.3767743662856892,0.07736324119662329,0.947,0.5403636983384068,1000 +Linear,Logistic,observational,True,4,0.9,0.4231666666666667,1.2357474079437374,0.7915034422252056,0.193,1.7654458413619594,1000 +Linear,Logistic,observational,True,4,0.95,0.5213333333333334,1.472483857452629,0.7915034422252056,0.277,1.9735390159048785,1000 +Linear,Logistic,observational,True,6,0.9,0.8928333333333334,1.0212995120870143,0.25694449841811784,0.89,1.4744168456619084,1000 +Linear,Logistic,observational,True,6,0.95,0.9460833333333334,1.216953428755113,0.25694449841811784,0.933,1.6462184670600197,1000 diff --git a/results/did/did_multi_eventstudy.csv b/results/did/did_multi_eventstudy.csv index 2977684..1c294b6 100644 --- a/results/did/did_multi_eventstudy.csv +++ b/results/did/did_multi_eventstudy.csv @@ -1,49 +1,49 @@ Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,experimental,False,1,0.9,0.2695,0.6664379809971152,0.5273245725879285,0.065,0.875601005136764,1000 -LGBM,LGBM,experimental,False,1,0.95,0.35933333333333334,0.7941098340189701,0.5273245725879285,0.123,0.993123730170564,1000 -LGBM,LGBM,experimental,False,4,0.9,0.398,0.5415820487655375,0.3712684690429321,0.2,0.7377555251101078,1000 -LGBM,LGBM,experimental,False,4,0.95,0.4905,0.6453348145154958,0.3712684690429321,0.303,0.8304953131373951,1000 -LGBM,LGBM,experimental,False,6,0.9,0.8975,0.5398952913714347,0.13395236250758816,0.89,0.7349862691616095,1000 -LGBM,LGBM,experimental,False,6,0.95,0.9498333333333334,0.6433249191126899,0.13395236250758816,0.943,0.8272174489329279,1000 -LGBM,LGBM,experimental,True,1,0.9,0.27066666666666667,0.6664609685957759,0.5268935068048141,0.073,0.8756450610322313,1000 -LGBM,LGBM,experimental,True,1,0.95,0.362,0.7941372254322411,0.5268935068048141,0.131,0.9924410930584766,1000 -LGBM,LGBM,experimental,True,4,0.9,0.3908333333333333,0.5414895331777497,0.37122437324957164,0.19,0.7376044296248573,1000 -LGBM,LGBM,experimental,True,4,0.95,0.4928333333333333,0.6452245753932411,0.37122437324957164,0.294,0.8300401173811606,1000 -LGBM,LGBM,experimental,True,6,0.9,0.8971666666666667,0.5397522456658546,0.13239266622447163,0.893,0.7343809360201817,1000 -LGBM,LGBM,experimental,True,6,0.95,0.9475,0.6431544696413897,0.13239266622447163,0.946,0.8273225484496115,1000 -LGBM,LGBM,observational,False,1,0.9,0.902,2.653721378497191,0.7070490800856716,0.933,3.6804836186191254,1000 -LGBM,LGBM,observational,False,1,0.95,0.958,3.162104056941675,0.7070490800856716,0.973,4.1198480194456915,1000 -LGBM,LGBM,observational,False,4,0.9,0.901,3.561000033087922,1.0237738258563158,0.94,4.872767022094824,1000 -LGBM,LGBM,observational,False,4,0.95,0.9645,4.243193254061008,1.0237738258563158,0.99,5.476875359796976,1000 -LGBM,LGBM,observational,False,6,0.9,0.932,2.0204226296120424,0.46987471691349253,0.953,2.8163439845272875,1000 -LGBM,LGBM,observational,False,6,0.95,0.9728333333333333,2.4074820535421066,0.46987471691349253,0.982,3.1491467155171455,1000 -LGBM,LGBM,observational,True,1,0.9,0.9123333333333333,1.07183187495378,0.2603956402406786,0.938,1.4925039393303094,1000 -LGBM,LGBM,observational,True,1,0.95,0.9661666666666666,1.2771664529718216,0.2603956402406786,0.969,1.6679753377448512,1000 -LGBM,LGBM,observational,True,4,0.9,0.936,1.3868740368469075,0.30530138471391266,0.942,1.9070687685924095,1000 -LGBM,LGBM,observational,True,4,0.95,0.9701666666666666,1.6525623427973326,0.30530138471391266,0.973,2.1382550302116794,1000 -LGBM,LGBM,observational,True,6,0.9,0.9173333333333333,0.9435499321525795,0.22352810241642249,0.928,1.3193862165661938,1000 -LGBM,LGBM,observational,True,6,0.95,0.96,1.1243090900810153,0.22352810241642249,0.965,1.4751970915806725,1000 -Linear,Linear,experimental,False,1,0.9,0.8038333333333334,0.21012774921404478,0.06511858439798791,0.723,0.29993875767609723,1000 -Linear,Linear,experimental,False,1,0.95,0.8825,0.25038265646487395,0.06511858439798791,0.819,0.3333237643577245,1000 -Linear,Linear,experimental,False,4,0.9,0.18866666666666665,0.9724995300651125,0.9445552734461886,0.035,1.2537683002940827,1000 -Linear,Linear,experimental,False,4,0.95,0.2515,1.1588046636358738,0.9445552734461886,0.059,1.4271667404908353,1000 -Linear,Linear,experimental,False,6,0.9,0.8888333333333334,0.9839291437722918,0.24625318596473464,0.885,1.265266070206284,1000 -Linear,Linear,experimental,False,6,0.95,0.941,1.172423888384033,0.24625318596473464,0.936,1.4400279208463234,1000 -Linear,Linear,experimental,True,1,0.9,0.8053333333333333,0.21013098816621933,0.06512604019501264,0.732,0.300014012964527,1000 -Linear,Linear,experimental,True,1,0.95,0.8816666666666666,0.2503865159144358,0.06512604019501264,0.824,0.333092537873174,1000 -Linear,Linear,experimental,True,4,0.9,0.18933333333333333,0.972518967504493,0.9443719698591424,0.037,1.2555177892281149,1000 -Linear,Linear,experimental,True,4,0.95,0.252,1.1588278247734445,0.9443719698591424,0.06,1.4274806534380786,1000 -Linear,Linear,experimental,True,6,0.9,0.8875,0.9839063096253385,0.24613101616711022,0.885,1.2651613665652979,1000 -Linear,Linear,experimental,True,6,0.95,0.9415,1.1723966798197494,0.24613101616711022,0.941,1.4397415352658391,1000 -Linear,Linear,observational,False,1,0.9,0.8916666666666666,0.22637212149559552,0.054683601823785656,0.891,0.3228630405513039,1000 -Linear,Linear,observational,False,1,0.95,0.9456666666666667,0.2697390199136439,0.054683601823785656,0.949,0.358552269863231,1000 -Linear,Linear,observational,False,4,0.9,0.3156666666666667,1.2872024456860576,0.9190699365216015,0.185,1.6388325707815383,1000 -Linear,Linear,observational,False,4,0.95,0.4146666666666667,1.5337963165952757,0.9190699365216015,0.262,1.8700830647518032,1000 -Linear,Linear,observational,False,6,0.9,0.8868333333333334,1.0372399263148084,0.26320551491048766,0.877,1.3311336840392576,1000 -Linear,Linear,observational,False,6,0.95,0.9406666666666667,1.2359476038435253,0.26320551491048766,0.943,1.5174947513822885,1000 -Linear,Linear,observational,True,1,0.9,0.8923333333333334,0.22495023196577907,0.05470760964666116,0.891,0.3207319286670861,1000 -Linear,Linear,observational,True,1,0.95,0.9435,0.2680447340375201,0.05470760964666116,0.94,0.35645975459527846,1000 -Linear,Linear,observational,True,4,0.9,0.3105,1.2870401885570502,0.9190767720949804,0.177,1.6393372900395775,1000 -Linear,Linear,observational,True,4,0.95,0.41433333333333333,1.533602975301024,0.9190767720949804,0.257,1.8708048686939287,1000 -Linear,Linear,observational,True,6,0.9,0.8815,1.028838214628254,0.26320130535902825,0.884,1.3218761748802963,1000 -Linear,Linear,observational,True,6,0.95,0.9405,1.225936346887698,0.26320130535902825,0.935,1.5055011269416756,1000 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.26816666666666666,0.6628004801492844,0.5217146610039974,0.063,0.8710835347098573,1000 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.356,0.7897754844217385,0.5217146610039974,0.106,0.9881445074099509,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.3793333333333333,0.5430545733393948,0.3833854158756925,0.176,0.73981228970138,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.4698333333333333,0.6470894357680017,0.3833854158756925,0.246,0.833246241332189,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.8915,0.5395169482604701,0.13534305762469595,0.897,0.7348860485385167,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9451666666666666,0.6428740954897616,0.13534305762469595,0.953,0.8259292882870245,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.265,0.6629226425912405,0.5221091420199984,0.062,0.8722109625410929,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.3551666666666667,0.7899210499496214,0.5221091420199984,0.114,0.9882984410974919,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.3763333333333333,0.5429965130173356,0.3828566010032573,0.182,0.739132671094994,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.4711666666666667,0.6470202526271421,0.3828566010032573,0.262,0.8314461004262811,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.8931666666666667,0.5395318624534728,0.13758915825744059,0.893,0.7341757740643098,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9463333333333334,0.6428918668468384,0.13758915825744059,0.946,0.8261153685402325,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.907,2.635755403048029,0.6906219348022667,0.931,3.658094889963508,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9618333333333333,3.140696276789984,0.6906219348022667,0.969,4.09528459799626,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.8978333333333334,3.5932112684720123,1.044176882449884,0.953,4.920160668033086,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9656666666666667,4.2815753083763015,1.044176882449884,0.982,5.521039436439466,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9356666666666666,2.037043783641883,0.45673572364777687,0.967,2.8380713672583395,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9768333333333333,2.4272873801354264,0.45673572364777687,0.99,3.173284984968741,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9186666666666666,1.058073826869023,0.2586744612407186,0.924,1.4747488796573536,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9613333333333334,1.2607727275351879,0.2586744612407186,0.965,1.649868845892826,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9368333333333334,1.4039435411486991,0.3103897362727539,0.943,1.9287327010063178,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9718333333333333,1.6729019116910515,0.3103897362727539,0.973,2.1651520912520748,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9153333333333333,0.9537908163093017,0.2248364423186676,0.927,1.3333350854360233,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9601666666666666,1.1365118562044807,0.2248364423186676,0.974,1.4916507037488054,1000 +Linear,Logistic,experimental,False,1,0.9,0.8085,0.21012919637470698,0.06454170881550668,0.737,0.3000298498259047,1000 +Linear,Logistic,experimental,False,1,0.95,0.8826666666666666,0.2503843808631616,0.06454170881550668,0.83,0.33342829695147896,1000 +Linear,Logistic,experimental,False,4,0.9,0.20066666666666666,0.9748536890898165,0.9456802642466463,0.041,1.2574525229653895,1000 +Linear,Logistic,experimental,False,4,0.95,0.26666666666666666,1.161609817132025,0.9456802642466463,0.074,1.4305039091925136,1000 +Linear,Logistic,experimental,False,6,0.9,0.8906666666666666,0.984266405297522,0.2447629940182398,0.885,1.2654037292644829,1000 +Linear,Logistic,experimental,False,6,0.95,0.9428333333333334,1.1728257602782801,0.2447629940182398,0.939,1.4411880110797295,1000 +Linear,Logistic,experimental,True,1,0.9,0.8095,0.21012765534784542,0.0646279391502723,0.733,0.29993071695819135,1000 +Linear,Logistic,experimental,True,1,0.95,0.8823333333333334,0.25038254461639875,0.0646279391502723,0.831,0.3331089494375711,1000 +Linear,Logistic,experimental,True,4,0.9,0.201,0.9747048561891234,0.9456904408930068,0.042,1.2571661797897207,1000 +Linear,Logistic,experimental,True,4,0.95,0.26666666666666666,1.1614324717924194,0.9456904408930068,0.075,1.4292891253140116,1000 +Linear,Logistic,experimental,True,6,0.9,0.8928333333333334,0.9843540987911897,0.24472186099367751,0.882,1.2667938573542303,1000 +Linear,Logistic,experimental,True,6,0.95,0.942,1.1729302535209924,0.24472186099367751,0.938,1.4425417792737238,1000 +Linear,Logistic,observational,False,1,0.9,0.8958333333333334,0.22595429878581444,0.05556582705974583,0.883,0.32193445273348237,1000 +Linear,Logistic,observational,False,1,0.95,0.9468333333333334,0.26924115344718413,0.05556582705974583,0.944,0.35775078341337774,1000 +Linear,Logistic,observational,False,4,0.9,0.325,1.2886928093204373,0.9238068984287002,0.175,1.6397882387716314,1000 +Linear,Logistic,observational,False,4,0.95,0.41933333333333334,1.535572194399471,0.9238068984287002,0.26,1.8701388333818019,1000 +Linear,Logistic,observational,False,6,0.9,0.8893333333333334,1.0302622288172734,0.2580572852924157,0.882,1.322817867501306,1000 +Linear,Logistic,observational,False,6,0.95,0.9411666666666666,1.2276331644514131,0.2580572852924157,0.934,1.5071984875399793,1000 +Linear,Logistic,observational,True,1,0.9,0.8921666666666667,0.22482298708967266,0.0554326928262382,0.882,0.3206665757666365,1000 +Linear,Logistic,observational,True,1,0.95,0.9446666666666667,0.2678931124158151,0.0554326928262382,0.941,0.3562766006057057,1000 +Linear,Logistic,observational,True,4,0.9,0.3243333333333333,1.2866810812005252,0.9246774960344024,0.177,1.6374108099983478,1000 +Linear,Logistic,observational,True,4,0.95,0.41833333333333333,1.5331750724932363,0.9246774960344024,0.258,1.869004453360348,1000 +Linear,Logistic,observational,True,6,0.9,0.8853333333333334,1.0254952185219397,0.25982703889124587,0.889,1.318157054156258,1000 +Linear,Logistic,observational,True,6,0.95,0.9436666666666667,1.2219529213345213,0.25982703889124587,0.94,1.502504660229367,1000 diff --git a/results/did/did_multi_group.csv b/results/did/did_multi_group.csv index d6d461b..e3035bc 100644 --- a/results/did/did_multi_group.csv +++ b/results/did/did_multi_group.csv @@ -1,49 +1,49 @@ Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,experimental,False,1,0.9,0.3723333333333333,0.712829716785441,0.5120534147173762,0.073,0.8873091560105003,1000 -LGBM,LGBM,experimental,False,1,0.95,0.4606666666666666,0.849388996757564,0.5120534147173762,0.122,1.0124899542984358,1000 -LGBM,LGBM,experimental,False,4,0.9,0.504,0.608382718738708,0.36202329008148987,0.219,0.7745653409497698,1000 -LGBM,LGBM,experimental,False,4,0.95,0.589,0.7249327222838707,0.36202329008148987,0.31,0.8776439618079263,1000 -LGBM,LGBM,experimental,False,6,0.9,0.8986666666666666,0.6042439878313273,0.14850794162785066,0.899,0.7675396748306148,1000 -LGBM,LGBM,experimental,False,6,0.95,0.9503333333333334,0.7200011202329313,0.14850794162785066,0.945,0.8700300494781059,1000 -LGBM,LGBM,experimental,True,1,0.9,0.3866666666666666,0.7129187295177013,0.5108468478621639,0.076,0.8873896060492763,1000 -LGBM,LGBM,experimental,True,1,0.95,0.4633333333333333,0.8494950619700162,0.5108468478621639,0.135,1.0122432445151552,1000 -LGBM,LGBM,experimental,True,4,0.9,0.5016666666666666,0.608263026948519,0.3616198619719443,0.223,0.7738821674571664,1000 -LGBM,LGBM,experimental,True,4,0.95,0.5966666666666667,0.7247901007191481,0.3616198619719443,0.308,0.877557935434779,1000 -LGBM,LGBM,experimental,True,6,0.9,0.8976666666666666,0.6041013460350053,0.14653383224031008,0.898,0.7676921917774191,1000 -LGBM,LGBM,experimental,True,6,0.95,0.947,0.7198311520491973,0.14653383224031008,0.954,0.8702372370829647,1000 -LGBM,LGBM,observational,False,1,0.9,0.9093333333333333,2.665571501346606,0.6783738614604253,0.937,3.372964798983418,1000 -LGBM,LGBM,observational,False,1,0.95,0.9673333333333334,3.1762243492380025,0.6783738614604253,0.98,3.825961439092819,1000 -LGBM,LGBM,observational,False,4,0.9,0.909,3.534531876327934,0.9946875737404597,0.935,4.445144884485254,1000 -LGBM,LGBM,observational,False,4,0.95,0.967,4.211654500012185,0.9946875737404597,0.976,5.055292173043325,1000 -LGBM,LGBM,observational,False,6,0.9,0.9416666666666667,2.122820536068101,0.4787787714125554,0.957,2.6963338440693168,1000 -LGBM,LGBM,observational,False,6,0.95,0.9766666666666667,2.5294966847881346,0.4787787714125554,0.986,3.0585559742251704,1000 -LGBM,LGBM,observational,True,1,0.9,0.921,1.1147256542022042,0.2672559046096273,0.934,1.4159126956485517,1000 -LGBM,LGBM,observational,True,1,0.95,0.966,1.3282775434118486,0.2672559046096273,0.968,1.6037050855151684,1000 -LGBM,LGBM,observational,True,4,0.9,0.94,1.422767817772445,0.30810115099970276,0.936,1.7975736984737825,1000 -LGBM,LGBM,observational,True,4,0.95,0.9706666666666667,1.6953324207728482,0.30810115099970276,0.972,2.041855425299719,1000 -LGBM,LGBM,observational,True,6,0.9,0.91,1.006741237739991,0.237584782868055,0.927,1.2840461885633785,1000 -LGBM,LGBM,observational,True,6,0.95,0.9603333333333334,1.1996061749145979,0.237584782868055,0.969,1.4521486558429435,1000 -Linear,Linear,experimental,False,1,0.9,0.809,0.2639472488371848,0.07836725122654463,0.751,0.33918155040956693,1000 -Linear,Linear,experimental,False,1,0.95,0.8853333333333334,0.3145125457139394,0.07836725122654463,0.833,0.3826104197586172,1000 -Linear,Linear,experimental,False,4,0.9,0.29933333333333334,1.0775733624029018,0.9144760793156517,0.031,1.3581931890994936,1000 -Linear,Linear,experimental,False,4,0.95,0.376,1.28400785723636,0.9144760793156517,0.066,1.544378342112665,1000 -Linear,Linear,experimental,False,6,0.9,0.8916666666666666,1.0855871936277797,0.2674422239505616,0.886,1.3663373525384621,1000 -Linear,Linear,experimental,False,6,0.95,0.9413333333333334,1.293556926114941,0.2674422239505616,0.947,1.554433869195187,1000 -Linear,Linear,experimental,True,1,0.9,0.809,0.26395101798234016,0.07841196950190694,0.751,0.3392834245796571,1000 -Linear,Linear,experimental,True,1,0.95,0.8856666666666666,0.3145170369274041,0.07841196950190694,0.832,0.38257509241292276,1000 -Linear,Linear,experimental,True,4,0.9,0.29733333333333334,1.0775450811401135,0.9139676817765434,0.033,1.3594859465438789,1000 -Linear,Linear,experimental,True,4,0.95,0.37666666666666665,1.283974158033225,0.9139676817765434,0.065,1.545038224259449,1000 -Linear,Linear,experimental,True,6,0.9,0.8946666666666666,1.0856219017160036,0.2672586672222071,0.896,1.366450981317095,1000 -Linear,Linear,experimental,True,6,0.95,0.942,1.2935982833529203,0.2672586672222071,0.952,1.5542102827957853,1000 -Linear,Linear,observational,False,1,0.9,0.8963333333333334,0.28417672167828534,0.0704315330527404,0.888,0.36504143473823225,1000 -Linear,Linear,observational,False,1,0.95,0.945,0.33861744936319155,0.0704315330527404,0.941,0.41189946472127525,1000 -Linear,Linear,observational,False,4,0.9,0.408,1.3757050815726335,0.8986741240953303,0.193,1.723360009232728,1000 -Linear,Linear,observational,False,4,0.95,0.5103333333333333,1.639253711728994,0.8986741240953303,0.289,1.9608953756229532,1000 -Linear,Linear,observational,False,6,0.9,0.8906666666666666,1.1373073172966048,0.284219716886748,0.88,1.4298799083888758,1000 -Linear,Linear,observational,False,6,0.95,0.9396666666666667,1.35518525462143,0.284219716886748,0.944,1.6242483562467662,1000 -Linear,Linear,observational,True,1,0.9,0.8976666666666666,0.28247229568225285,0.07026934090791014,0.889,0.3628745212929247,1000 -Linear,Linear,observational,True,1,0.95,0.9476666666666667,0.33658650052263794,0.07026934090791014,0.945,0.40971246171295345,1000 -Linear,Linear,observational,True,4,0.9,0.4056666666666666,1.3749682398133307,0.8988135221197031,0.198,1.7216124241871145,1000 -Linear,Linear,observational,True,4,0.95,0.505,1.638375710618819,0.8988135221197031,0.282,1.959504408369747,1000 -Linear,Linear,observational,True,6,0.9,0.8893333333333334,1.1272233554666016,0.2838441687177051,0.879,1.416566966857148,1000 -Linear,Linear,observational,True,6,0.95,0.943,1.3431694729832098,0.2838441687177051,0.945,1.6114267967021605,1000 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.37966666666666665,0.7094842843054401,0.5063966456271116,0.07,0.8833695236671398,1000 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.463,0.8454026680860791,0.5063966456271116,0.119,1.0075828733731322,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.48833333333333334,0.6093983296457081,0.3746967239327951,0.185,0.7748043075861907,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.5676666666666667,0.7261428973215824,0.3746967239327951,0.257,0.8788079154824321,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.8953333333333334,0.6039293782365326,0.15156143782863715,0.905,0.7671387444581327,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9506666666666667,0.7196262397785951,0.15156143782863715,0.952,0.8709721345495236,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.3783333333333333,0.7096389669594401,0.5063175240399332,0.066,0.8838830720898343,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.4603333333333333,0.8455869838366757,0.5063175240399332,0.124,1.0089102464690358,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.48533333333333334,0.609259041148495,0.37470955013293855,0.19,0.7754703791594748,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.5703333333333332,0.7259769248401865,0.37470955013293855,0.27,0.8784156068436955,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.8876666666666666,0.6038826619307712,0.15256760709225262,0.904,0.7681471625685046,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.948,0.7195705738668794,0.15256760709225262,0.954,0.8695993470187038,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.915,2.6608678733015485,0.688154090500346,0.931,3.366696329353361,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9656666666666667,3.1706196307305743,0.688154090500346,0.97,3.8192561069995286,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.903,3.556747699127985,0.9996416099478588,0.942,4.472327607652934,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.969,4.238126285623724,0.9996416099478588,0.984,5.088003820636524,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9373333333333334,2.1290784823926696,0.46227527065430596,0.956,2.703189636335836,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9743333333333334,2.5369534877597597,0.46227527065430596,0.991,3.0641146657072404,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.91,1.0977119557447321,0.2651109928804438,0.939,1.3939185973124866,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.968,1.3080044712830703,0.2651109928804438,0.981,1.580610122013623,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9316666666666666,1.4368875613066505,0.30987690101867293,0.952,1.8152404100048158,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.975,1.7121571329201994,0.30987690101867293,0.98,2.059654079760943,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.908,1.0154733246915484,0.23971463199529405,0.929,1.2938281773467855,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.96,1.210011098279508,0.23971463199529405,0.972,1.4652849487780493,1000 +Linear,Logistic,experimental,False,1,0.9,0.8176666666666667,0.2639777067863099,0.07735852107118578,0.764,0.3393389889591518,1000 +Linear,Logistic,experimental,False,1,0.95,0.891,0.3145488385988198,0.07735852107118578,0.839,0.3826999999595859,1000 +Linear,Logistic,experimental,False,4,0.9,0.31,1.0799528007716814,0.9173788544387992,0.045,1.361708244102074,1000 +Linear,Logistic,experimental,False,4,0.95,0.385,1.2868431329288754,0.9173788544387992,0.073,1.5474903461640912,1000 +Linear,Logistic,experimental,False,6,0.9,0.8963333333333334,1.0864824219772902,0.2703183844096557,0.893,1.3666259596934611,1000 +Linear,Logistic,experimental,False,6,0.95,0.949,1.2946236564879237,0.2703183844096557,0.943,1.5549441788053244,1000 +Linear,Logistic,experimental,True,1,0.9,0.8163333333333334,0.2639754882372008,0.07738204276505496,0.765,0.339331857870017,1000 +Linear,Logistic,experimental,True,1,0.95,0.8906666666666666,0.3145461950345047,0.07738204276505496,0.844,0.3833030006646118,1000 +Linear,Logistic,experimental,True,4,0.9,0.3103333333333333,1.0798559231989837,0.9168941023520879,0.046,1.3607552807065624,1000 +Linear,Logistic,experimental,True,4,0.95,0.3873333333333333,1.2867276961810175,0.9168941023520879,0.074,1.5475927725689025,1000 +Linear,Logistic,experimental,True,6,0.9,0.8986666666666666,1.0865640980568139,0.27054317418995893,0.896,1.3676856739628556,1000 +Linear,Logistic,experimental,True,6,0.95,0.95,1.2947209795394352,0.27054317418995893,0.941,1.5556519546992444,1000 +Linear,Logistic,observational,False,1,0.9,0.8956666666666666,0.28367316450513197,0.0700767685399924,0.903,0.3642537051104781,1000 +Linear,Logistic,observational,False,1,0.95,0.9523333333333334,0.3380174239825948,0.0700767685399924,0.948,0.4113211504413545,1000 +Linear,Logistic,observational,False,4,0.9,0.408,1.3776824648934105,0.9052340667640746,0.183,1.7236892476868426,1000 +Linear,Logistic,observational,False,4,0.95,0.5046666666666666,1.641609909282898,0.9052340667640746,0.277,1.964267689737536,1000 +Linear,Logistic,observational,False,6,0.9,0.8903333333333334,1.1309252917212975,0.282843171368907,0.89,1.4213783682706032,1000 +Linear,Logistic,observational,False,6,0.95,0.9493333333333334,1.3475806021033823,0.282843171368907,0.947,1.617450746310541,1000 +Linear,Logistic,observational,True,1,0.9,0.8876666666666666,0.28210363817731876,0.0702387016495634,0.9,0.36227342165859067,1000 +Linear,Logistic,observational,True,1,0.95,0.9513333333333334,0.3361472180111355,0.0702387016495634,0.945,0.40929779552981904,1000 +Linear,Logistic,observational,True,4,0.9,0.4043333333333333,1.3755156336478902,0.904918936330756,0.185,1.7226484388527301,1000 +Linear,Logistic,observational,True,4,0.95,0.497,1.6390279706032433,0.904918936330756,0.286,1.9605550323619565,1000 +Linear,Logistic,observational,True,6,0.9,0.892,1.125633507574361,0.2835718025458131,0.899,1.414839244504656,1000 +Linear,Logistic,observational,True,6,0.95,0.9473333333333334,1.341275052374208,0.2835718025458131,0.949,1.6097715181154,1000 diff --git a/results/did/did_multi_metadata.csv b/results/did/did_multi_metadata.csv index 191c4eb..86381a7 100644 --- a/results/did/did_multi_metadata.csv +++ b/results/did/did_multi_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.10.dev0,DIDMultiCoverageSimulation,2025-05-23 09:01,152.65548847913743,3.12.9,scripts/did/did_pa_multi_config.yml +0.10.0,DIDMultiCoverageSimulation,2025-06-03 09:09,162.553562772274,3.12.9,scripts/did/did_pa_multi_config.yml diff --git a/results/did/did_multi_time.csv b/results/did/did_multi_time.csv index 333f774..3dda54d 100644 --- a/results/did/did_multi_time.csv +++ b/results/did/did_multi_time.csv @@ -1,49 +1,49 @@ Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,experimental,False,1,0.9,0.105,0.6770474680143006,0.5882187359928296,0.06,0.8037202643084942,1000 -LGBM,LGBM,experimental,False,1,0.95,0.17066666666666666,0.8067518175410349,0.5882187359928296,0.118,0.926841610902776,1000 -LGBM,LGBM,experimental,False,4,0.9,0.236,0.5444213002720629,0.41426795329176225,0.178,0.6607984097007097,1000 -LGBM,LGBM,experimental,False,4,0.95,0.3276666666666666,0.6487179913554645,0.41426795329176225,0.267,0.7587562686524103,1000 -LGBM,LGBM,experimental,False,6,0.9,0.8943333333333334,0.53982673853458,0.1342938994875208,0.908,0.6564456680014384,1000 -LGBM,LGBM,experimental,False,6,0.95,0.95,0.6432432333693072,0.1342938994875208,0.951,0.7533485914407448,1000 -LGBM,LGBM,experimental,True,1,0.9,0.104,0.6773164375641183,0.5871963271333144,0.063,0.8038330391648153,1000 -LGBM,LGBM,experimental,True,1,0.95,0.17566666666666667,0.8070723145274795,0.5871963271333144,0.136,0.9270642436123595,1000 -LGBM,LGBM,experimental,True,4,0.9,0.2313333333333333,0.5443770179339803,0.414059062792942,0.171,0.6603179410869597,1000 -LGBM,LGBM,experimental,True,4,0.95,0.327,0.6486652256951217,0.414059062792942,0.258,0.7584758454668876,1000 -LGBM,LGBM,experimental,True,6,0.9,0.9013333333333333,0.5395668449941098,0.13154818699839163,0.895,0.6563984364887405,1000 -LGBM,LGBM,experimental,True,6,0.95,0.9493333333333334,0.6429335511150388,0.13154818699839163,0.942,0.7529872439559875,1000 -LGBM,LGBM,observational,False,1,0.9,0.8973333333333333,2.8900944626182756,0.7577316621702538,0.915,3.57007703127756,1000 -LGBM,LGBM,observational,False,1,0.95,0.958,3.4437599588413588,0.7577316621702538,0.968,4.083723276817505,1000 -LGBM,LGBM,observational,False,4,0.9,0.8883333333333334,3.944278944663819,1.1508934356199159,0.921,4.817695497037129,1000 -LGBM,LGBM,observational,False,4,0.95,0.959,4.699898246173131,1.1508934356199159,0.981,5.525753606365474,1000 -LGBM,LGBM,observational,False,6,0.9,0.9346666666666666,2.007004968725184,0.4621685573032238,0.943,2.5048024891920218,1000 -LGBM,LGBM,observational,False,6,0.95,0.973,2.391493924468424,0.4621685573032238,0.976,2.8553234091559982,1000 -LGBM,LGBM,observational,True,1,0.9,0.9203333333333333,1.1214028688438904,0.2681238910047797,0.932,1.3877940365894852,1000 -LGBM,LGBM,observational,True,1,0.95,0.966,1.336233935397319,0.2681238910047797,0.965,1.5830862634102072,1000 -LGBM,LGBM,observational,True,4,0.9,0.946,1.4738907100586922,0.3155685506978381,0.947,1.8023292945303246,1000 -LGBM,LGBM,observational,True,4,0.95,0.975,1.756249104193653,0.3155685506978381,0.978,2.0652293790201095,1000 -LGBM,LGBM,observational,True,6,0.9,0.9036666666666666,0.9243181245460811,0.22145829614971912,0.904,1.1513913386950982,1000 -LGBM,LGBM,observational,True,6,0.95,0.9506666666666667,1.101392977881901,0.22145829614971912,0.964,1.312498872874679,1000 -Linear,Linear,experimental,False,1,0.9,0.785,0.24421028738791545,0.07700598814984021,0.726,0.31283981302906455,1000 -Linear,Linear,experimental,False,1,0.95,0.8656666666666666,0.29099450558503215,0.07700598814984021,0.822,0.3536648083533155,1000 -Linear,Linear,experimental,False,4,0.9,0.029,0.9657408497851493,1.0738907262320931,0.024,1.1066254403295672,1000 -Linear,Linear,experimental,False,4,0.95,0.05633333333333333,1.1507511993551036,1.0738907262320931,0.042,1.2857511031369329,1000 -Linear,Linear,experimental,False,6,0.9,0.891,0.9640961231527081,0.2403665468723713,0.887,1.1081712902827776,1000 -Linear,Linear,experimental,False,6,0.95,0.944,1.1487913866938562,0.2403665468723713,0.945,1.2857969535196567,1000 -Linear,Linear,experimental,True,1,0.9,0.7856666666666666,0.24421161220168236,0.07699427764290885,0.725,0.312745321017087,1000 -Linear,Linear,experimental,True,1,0.95,0.866,0.2909960841980021,0.07699427764290885,0.82,0.35353627547158184,1000 -Linear,Linear,experimental,True,4,0.9,0.029333333333333333,0.9658114690308013,1.073411274232068,0.025,1.1072936001699796,1000 -Linear,Linear,experimental,True,4,0.95,0.056,1.1508353473764386,1.073411274232068,0.047,1.2871025490442616,1000 -Linear,Linear,experimental,True,6,0.9,0.8933333333333334,0.964156052938097,0.23997981769673798,0.891,1.1072565217168384,1000 -Linear,Linear,experimental,True,6,0.95,0.944,1.1488627974376686,0.23997981769673798,0.947,1.287096789672995,1000 -Linear,Linear,observational,False,1,0.9,0.889,0.2746570982471215,0.06835594688685424,0.877,0.35201642493170093,1000 -Linear,Linear,observational,False,1,0.95,0.9386666666666666,0.32727411840307125,0.06835594688685424,0.932,0.39803367647348065,1000 -Linear,Linear,observational,False,4,0.9,0.16033333333333336,1.3480635210832488,1.0552310812916714,0.131,1.518276214952089,1000 -Linear,Linear,observational,False,4,0.95,0.24266666666666667,1.6063167608976374,1.0552310812916714,0.211,1.7707505491415911,1000 -Linear,Linear,observational,False,6,0.9,0.8903333333333334,1.0186244057373655,0.2564255464027685,0.88,1.171669804736589,1000 -Linear,Linear,observational,False,6,0.95,0.9423333333333334,1.2137658429333622,0.2564255464027685,0.942,1.359505036566505,1000 -Linear,Linear,observational,True,1,0.9,0.8853333333333334,0.27250071547513943,0.06816094516072957,0.876,0.349158428206522,1000 -Linear,Linear,observational,True,1,0.95,0.9393333333333334,0.3247046298475451,0.06816094516072957,0.927,0.39461799076066073,1000 -Linear,Linear,observational,True,4,0.9,0.15733333333333335,1.348359701155594,1.0540447813311582,0.131,1.518475986344912,1000 -Linear,Linear,observational,True,4,0.95,0.24566666666666664,1.6066696812215027,1.0540447813311582,0.213,1.7697898872953792,1000 -Linear,Linear,observational,True,6,0.9,0.886,1.007087091732094,0.2561924661932343,0.886,1.1588097256348897,1000 -Linear,Linear,observational,True,6,0.95,0.9426666666666667,1.2000182853646246,0.2561924661932343,0.938,1.3446481778510444,1000 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.09633333333333333,0.6730703224735359,0.5813847440636578,0.06,0.7994113182366382,1000 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.163,0.8020127563300868,0.5813847440636578,0.109,0.9222994395169547,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.20866666666666667,0.545809228523173,0.4318166485465945,0.148,0.6621126753160614,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.29,0.6503718098720356,0.4318166485465945,0.226,0.7607781957690535,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.8883333333333334,0.539451744339498,0.1374713900065768,0.888,0.6559678253242532,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.943,0.6427964002257806,0.1374713900065768,0.942,0.7532136346784374,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.095,0.673215344660322,0.5816575606523053,0.066,0.7987689454841812,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.16966666666666666,0.8021855609240048,0.5816575606523053,0.116,0.9211448037237728,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.212,0.5457365545641853,0.43118597517736307,0.158,0.662287763310214,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.29633333333333334,0.6502852135087509,0.43118597517736307,0.223,0.7601307241685876,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.882,0.5395052157780649,0.13917064701840512,0.889,0.6561053221874785,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.938,0.6428601153747003,0.13917064701840512,0.947,0.753120590966105,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.8976666666666666,2.8641489580508317,0.7473265850366615,0.92,3.5454942167899373,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.962,3.41284398329212,0.7473265850366615,0.97,4.046408625403042,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.8876666666666666,3.9648529818754157,1.1487073320114483,0.918,4.845766161863059,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9593333333333334,4.724413723593485,1.1487073320114483,0.986,5.548216313166135,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.938,2.0148113447449116,0.43840143925357783,0.955,2.5144593978114727,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9783333333333334,2.4007957952232104,0.43840143925357783,0.99,2.861410408432865,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9076666666666666,1.1003301479666492,0.27099358031240817,0.913,1.3627159220412297,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9583333333333334,1.311124239738742,0.27099358031240817,0.97,1.5572874751641161,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9333333333333333,1.4854173027590014,0.3229586888066522,0.95,1.8150130378533484,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.976,1.7699838865395685,0.3229586888066522,0.978,2.0803451880507104,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9163333333333333,0.9353518863595044,0.219408785264352,0.912,1.1653988211882675,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9603333333333334,1.1145405160056325,0.219408785264352,0.959,1.3283422144863115,1000 +Linear,Logistic,experimental,False,1,0.9,0.7966666666666666,0.24428496421942136,0.07478155567895448,0.728,0.3129228235483047,1000 +Linear,Logistic,experimental,False,1,0.95,0.8733333333333334,0.2910834885181229,0.07478155567895448,0.833,0.35375833173558197,1000 +Linear,Logistic,experimental,False,4,0.9,0.038,0.9678137745111298,1.0814213490525186,0.03,1.1087428056263964,1000 +Linear,Logistic,experimental,False,4,0.95,0.06666666666666667,1.1532212415150949,1.0814213490525186,0.054,1.2896096218626036,1000 +Linear,Logistic,experimental,False,6,0.9,0.8883333333333334,0.9645120733919993,0.2398038156236055,0.884,1.1089972839130895,1000 +Linear,Logistic,experimental,False,6,0.95,0.9386666666666666,1.1492870219741105,0.2398038156236055,0.939,1.2860092104213576,1000 +Linear,Logistic,experimental,True,1,0.9,0.7976666666666666,0.244274323547519,0.0748075511921299,0.735,0.31288102428256387,1000 +Linear,Logistic,experimental,True,1,0.95,0.8716666666666666,0.2910708093755182,0.0748075511921299,0.833,0.35371101065661276,1000 +Linear,Logistic,experimental,True,4,0.9,0.03833333333333333,0.9676295529475489,1.0812355403537053,0.03,1.10872071383113,1000 +Linear,Logistic,experimental,True,4,0.95,0.067,1.1530017279827793,1.0812355403537053,0.054,1.289018545293176,1000 +Linear,Logistic,experimental,True,6,0.9,0.885,0.964486553866753,0.23961465238247193,0.887,1.109074934480792,1000 +Linear,Logistic,experimental,True,6,0.95,0.9383333333333334,1.1492566135842297,0.23961465238247193,0.933,1.2873487345102677,1000 +Linear,Logistic,observational,False,1,0.9,0.8873333333333334,0.2738660718613766,0.0672792367714224,0.875,0.35107388214992413,1000 +Linear,Logistic,observational,False,1,0.95,0.9376666666666666,0.32633155232820765,0.0672792367714224,0.938,0.39679498726255896,1000 +Linear,Logistic,observational,False,4,0.9,0.16433333333333333,1.3471569564681332,1.0662036471383771,0.133,1.5189433786007114,1000 +Linear,Logistic,observational,False,4,0.95,0.248,1.6052365225310308,1.0662036471383771,0.203,1.7701614407109725,1000 +Linear,Logistic,observational,False,6,0.9,0.8823333333333334,1.0106046290695898,0.25297949046206,0.886,1.1632254064709615,1000 +Linear,Logistic,observational,False,6,0.95,0.9396666666666667,1.20420968962261,0.25297949046206,0.941,1.3507861334703333,1000 +Linear,Logistic,observational,True,1,0.9,0.8836666666666666,0.27183527948043773,0.06710802675280161,0.87,0.3482504195565255,1000 +Linear,Logistic,observational,True,1,0.95,0.9373333333333334,0.3239117139538379,0.06710802675280161,0.934,0.3934241047155358,1000 +Linear,Logistic,observational,True,4,0.9,0.17166666666666666,1.3476603584428708,1.0647967745035234,0.133,1.5187476579231214,1000 +Linear,Logistic,observational,True,4,0.95,0.247,1.6058363629813088,1.0647967745035234,0.207,1.7714708628832339,1000 +Linear,Logistic,observational,True,6,0.9,0.889,1.0055746633220144,0.25465275272035065,0.884,1.1560383777095113,1000 +Linear,Logistic,observational,True,6,0.95,0.9386666666666666,1.1982161157585394,0.25465275272035065,0.939,1.3436875509579689,1000 diff --git a/results/did/did_pa_atte_coverage_metadata.csv b/results/did/did_pa_atte_coverage_metadata.csv index 5a18477..961444b 100644 --- a/results/did/did_pa_atte_coverage_metadata.csv +++ b/results/did/did_pa_atte_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,did_pa_atte_coverage.py,2025-05-22 14:30:49,10714.369587659836,3.12.3 +0.11.dev0,did_pa_atte_coverage.py,2025-06-02 15:10:48,10769.977479457855,3.12.3 diff --git a/results/did/did_pa_multi_config.yml b/results/did/did_pa_multi_config.yml index fa87158..ed4e23a 100644 --- a/results/did/did_pa_multi_config.yml +++ b/results/did/did_pa_multi_config.yml @@ -1,7 +1,8 @@ -confidence_parameters: - level: - - 0.95 - - 0.9 +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 dgp_parameters: DGP: - 1 @@ -9,115 +10,34 @@ dgp_parameters: - 6 n_obs: - 2000 -dml_parameters: - in_sample_normalization: - - true - - false - learners: - - ml_g: !!python/tuple - - Linear - - !!python/object:sklearn.linear_model._base.LinearRegression - _sklearn_version: 1.5.2 - copy_X: true - fit_intercept: true - n_jobs: null - positive: false - ml_m: !!python/tuple - - Linear - - !!python/object:sklearn.linear_model._logistic.LogisticRegression - C: 1.0 - _sklearn_version: 1.5.2 - class_weight: null - dual: false - fit_intercept: true - intercept_scaling: 1 - l1_ratio: null - max_iter: 100 - multi_class: deprecated - n_jobs: null - penalty: l2 - random_state: null - solver: lbfgs - tol: 0.0001 - verbose: 0 - warm_start: false - - ml_g: !!python/tuple - - LGBM - - !!python/object:lightgbm.sklearn.LGBMRegressor - _Booster: null - _best_iteration: -1 - _best_score: {} - _class_map: null - _class_weight: null - _classes: null - _evals_result: {} - _n_classes: -1 - _n_features: -1 - _n_features_in: -1 - _objective: null - _other_params: - verbose: -1 - boosting_type: gbdt - class_weight: null - colsample_bytree: 1.0 - importance_type: split - learning_rate: 0.02 - max_depth: -1 - min_child_samples: 20 - min_child_weight: 0.001 - min_split_gain: 0.0 +learner_definitions: + linear: &id001 + name: Linear + logistic: &id002 + name: Logistic + lgbmr: &id003 + name: LGBM Regr. + params: n_estimators: 500 - n_jobs: 1 - num_leaves: 31 - objective: null - random_state: null - reg_alpha: 0.0 - reg_lambda: 0.0 - subsample: 1.0 - subsample_for_bin: 200000 - subsample_freq: 0 - verbose: -1 - ml_m: !!python/tuple - - LGBM - - !!python/object:lightgbm.sklearn.LGBMClassifier - _Booster: null - _best_iteration: -1 - _best_score: {} - _class_map: null - _class_weight: null - _classes: null - _evals_result: {} - _n_classes: -1 - _n_features: -1 - _n_features_in: -1 - _objective: null - _other_params: - verbose: -1 - boosting_type: gbdt - class_weight: null - colsample_bytree: 1.0 - importance_type: split learning_rate: 0.02 - max_depth: -1 - min_child_samples: 20 - min_child_weight: 0.001 - min_split_gain: 0.0 + lgbmc: &id004 + name: LGBM Clas. + params: n_estimators: 500 - n_jobs: 1 - num_leaves: 31 - objective: null - random_state: null - reg_alpha: 0.0 - reg_lambda: 0.0 - subsample: 1.0 - subsample_for_bin: 200000 - subsample_freq: 0 - verbose: -1 + learning_rate: 0.02 +dml_parameters: + learners: + - ml_g: *id001 + ml_m: *id002 + - ml_g: *id003 + ml_m: *id004 score: - observational - experimental -simulation_parameters: - max_runtime: 19800 - n_jobs: -2 - random_seed: 42 - repetitions: 1000 + in_sample_normalization: + - true + - false +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/rdd/rdd_fuzzy_config.yml b/results/rdd/rdd_fuzzy_config.yml new file mode 100644 index 0000000..1c010bd --- /dev/null +++ b/results/rdd/rdd_fuzzy_config.yml @@ -0,0 +1,63 @@ +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + n_obs: + - 2000 + fuzzy: + - true + cutoff: + - 0.0 +learner_definitions: + lgbmr: &id001 + name: LGBM Regr. + params: + n_estimators: 200 + learning_rate: 0.02 + max_depth: 5 + lgbmc: &id002 + name: LGBM Clas. + params: + n_estimators: 200 + learning_rate: 0.02 + max_depth: 5 + global_linear: &id003 + name: Global Linear + global_logistic: &id004 + name: Global Logistic + local_linear: &id005 + name: Linear + local_logistic: &id006 + name: Logistic + stacked_reg: &id007 + name: Stacked Regr. + params: + n_estimators: 200 + learning_rate: 0.02 + max_depth: 5 + stacked_cls: &id008 + name: Stacked Clas. + params: + n_estimators: 200 + learning_rate: 0.02 + max_depth: 5 +dml_parameters: + fs_specification: + - cutoff + - cutoff and score + - interacted cutoff and score + learners: + - ml_g: *id001 + ml_m: *id002 + - ml_g: *id003 + ml_m: *id004 + - ml_g: *id005 + ml_m: *id006 + - ml_g: *id007 + ml_m: *id008 +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/rdd/rdd_fuzzy_coverage.csv b/results/rdd/rdd_fuzzy_coverage.csv index ebec18a..a9a6dea 100644 --- a/results/rdd/rdd_fuzzy_coverage.csv +++ b/results/rdd/rdd_fuzzy_coverage.csv @@ -1,99 +1,27 @@ -Method,fs specification,Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -rdflex,cutoff,Global linear,Global linear,0.9,0.928,10.556606789284906,2.56683549934152,250 -rdflex,cutoff,Global linear,Global linear,0.95,0.964,12.578972844104541,2.56683549934152,250 -rdflex,cutoff,Global linear,LGBM,0.9,0.932,10.775966548524359,2.6575671123131777,250 -rdflex,cutoff,Global linear,LGBM,0.95,0.976,12.840356119018518,2.6575671123131777,250 -rdflex,cutoff,Global linear,Linear,0.9,0.928,10.680654195730742,2.6095109173162823,250 -rdflex,cutoff,Global linear,Linear,0.95,0.964,12.726784445711942,2.6095109173162823,250 -rdflex,cutoff,Global linear,Stacked,0.9,0.932,10.571620600845328,2.6448751082127377,250 -rdflex,cutoff,Global linear,Stacked,0.95,0.96,12.596862904014394,2.6448751082127377,250 -rdflex,cutoff,LGBM,Global linear,0.9,0.94,2.038612360942403,0.46302745914972104,250 -rdflex,cutoff,LGBM,Global linear,0.95,0.968,2.4291564552711176,0.46302745914972104,250 -rdflex,cutoff,LGBM,LGBM,0.9,0.94,2.0417477871653618,0.4912050569704094,250 -rdflex,cutoff,LGBM,LGBM,0.95,0.976,2.4328925460529893,0.4912050569704094,250 -rdflex,cutoff,LGBM,Linear,0.9,0.96,2.088603350877394,0.47982216259182514,250 -rdflex,cutoff,LGBM,Linear,0.95,0.992,2.488724393851574,0.47982216259182514,250 -rdflex,cutoff,LGBM,Stacked,0.9,0.916,2.0184810970572267,0.46215722170125356,250 -rdflex,cutoff,LGBM,Stacked,0.95,0.972,2.405168574810687,0.46215722170125356,250 -rdflex,cutoff,Linear,Global linear,0.9,0.928,10.564695732971051,2.553945752977564,250 -rdflex,cutoff,Linear,Global linear,0.95,0.968,12.58861141500109,2.553945752977564,250 -rdflex,cutoff,Linear,LGBM,0.9,0.928,10.777243710702589,2.650598511347432,250 -rdflex,cutoff,Linear,LGBM,0.95,0.972,12.841877951618534,2.650598511347432,250 -rdflex,cutoff,Linear,Linear,0.9,0.92,10.74105034617443,2.6072040698667376,250 -rdflex,cutoff,Linear,Linear,0.95,0.968,12.798750897762686,2.6072040698667376,250 -rdflex,cutoff,Linear,Stacked,0.9,0.932,10.404127851046006,2.505787762202565,250 -rdflex,cutoff,Linear,Stacked,0.95,0.968,12.397282982798743,2.505787762202565,250 -rdflex,cutoff,Stacked,Global linear,0.9,0.928,2.100685938954007,0.4945555538677959,250 -rdflex,cutoff,Stacked,Global linear,0.95,0.972,2.50312168555107,0.4945555538677959,250 -rdflex,cutoff,Stacked,LGBM,0.9,0.956,2.003482953646103,0.4641511546138578,250 -rdflex,cutoff,Stacked,LGBM,0.95,0.992,2.3872971846522506,0.4641511546138578,250 -rdflex,cutoff,Stacked,Linear,0.9,0.944,2.11736656727605,0.46071259221608185,250 -rdflex,cutoff,Stacked,Linear,0.95,0.984,2.522997880134595,0.46071259221608185,250 -rdflex,cutoff,Stacked,Stacked,0.9,0.936,2.0245073903258874,0.44210382922098473,250 -rdflex,cutoff,Stacked,Stacked,0.95,0.976,2.412349346140925,0.44210382922098473,250 -rdflex,cutoff and score,Global linear,Global linear,0.9,0.928,10.544420005750476,2.567803474303041,250 -rdflex,cutoff and score,Global linear,Global linear,0.95,0.964,12.564451395859244,2.567803474303041,250 -rdflex,cutoff and score,Global linear,LGBM,0.9,0.928,10.805365747954351,2.6437437844814258,250 -rdflex,cutoff and score,Global linear,LGBM,0.95,0.972,12.875387425824758,2.6437437844814258,250 -rdflex,cutoff and score,Global linear,Linear,0.9,0.928,10.699263774918355,2.5999147658504103,250 -rdflex,cutoff and score,Global linear,Linear,0.95,0.964,12.748959127019463,2.5999147658504103,250 -rdflex,cutoff and score,Global linear,Stacked,0.9,0.932,10.82713502969946,2.6270982998447745,250 -rdflex,cutoff and score,Global linear,Stacked,0.95,0.972,12.901327124950933,2.6270982998447745,250 -rdflex,cutoff and score,LGBM,Global linear,0.9,0.956,2.1656416896389774,0.4963137476876054,250 -rdflex,cutoff and score,LGBM,Global linear,0.95,0.984,2.5805212363957613,0.4963137476876054,250 -rdflex,cutoff and score,LGBM,LGBM,0.9,0.94,2.201449781464929,0.5341193634721544,250 -rdflex,cutoff and score,LGBM,LGBM,0.95,0.972,2.6231892095114255,0.5341193634721544,250 -rdflex,cutoff and score,LGBM,Linear,0.9,0.952,2.1894694844073546,0.4635302319336836,250 -rdflex,cutoff and score,LGBM,Linear,0.95,0.98,2.6089138050789615,0.4635302319336836,250 -rdflex,cutoff and score,LGBM,Stacked,0.9,0.964,2.1361906429140674,0.5017184797294254,250 -rdflex,cutoff and score,LGBM,Stacked,0.95,0.984,2.545428149727124,0.5017184797294254,250 -rdflex,cutoff and score,Linear,Global linear,0.9,0.92,10.636843214150689,2.585401553398786,250 -rdflex,cutoff and score,Linear,Global linear,0.95,0.968,12.67458044128427,2.585401553398786,250 -rdflex,cutoff and score,Linear,LGBM,0.9,0.936,10.869584147939532,2.6903868750455233,250 -rdflex,cutoff and score,Linear,LGBM,0.95,0.98,12.95190836911928,2.6903868750455233,250 -rdflex,cutoff and score,Linear,Linear,0.9,0.912,10.715620665219745,2.5870008277605034,250 -rdflex,cutoff and score,Linear,Linear,0.95,0.968,12.76844956395835,2.5870008277605034,250 -rdflex,cutoff and score,Linear,Stacked,0.9,0.92,10.814063241507966,2.6455094849616714,250 -rdflex,cutoff and score,Linear,Stacked,0.95,0.964,12.88575112861359,2.6455094849616714,250 -rdflex,cutoff and score,Stacked,Global linear,0.9,0.932,2.1726374228239367,0.4855323537680247,250 -rdflex,cutoff and score,Stacked,Global linear,0.95,0.976,2.5888571666349667,0.4855323537680247,250 -rdflex,cutoff and score,Stacked,LGBM,0.9,0.952,2.14733586500187,0.5004783937020142,250 -rdflex,cutoff and score,Stacked,LGBM,0.95,0.972,2.5587085009595185,0.5004783937020142,250 -rdflex,cutoff and score,Stacked,Linear,0.9,0.948,2.202424304512329,0.49388698262577957,250 -rdflex,cutoff and score,Stacked,Linear,0.95,0.992,2.6243504253446837,0.49388698262577957,250 -rdflex,cutoff and score,Stacked,Stacked,0.9,0.956,2.23318991538815,0.4972797313578032,250 -rdflex,cutoff and score,Stacked,Stacked,0.95,0.98,2.6610099118126316,0.4972797313578032,250 -rdflex,interacted cutoff and score,Global linear,Global linear,0.9,0.932,10.545804244171494,2.539582402518461,250 -rdflex,interacted cutoff and score,Global linear,Global linear,0.95,0.964,12.566100817672075,2.539582402518461,250 -rdflex,interacted cutoff and score,Global linear,LGBM,0.9,0.932,10.884267411835014,2.650211237977455,250 -rdflex,interacted cutoff and score,Global linear,LGBM,0.95,0.972,12.969404557192851,2.650211237977455,250 -rdflex,interacted cutoff and score,Global linear,Linear,0.9,0.936,10.674123116042953,2.5790433126760175,250 -rdflex,interacted cutoff and score,Global linear,Linear,0.95,0.964,12.719002184264186,2.5790433126760175,250 -rdflex,interacted cutoff and score,Global linear,Stacked,0.9,0.932,10.626731756603398,2.6159768660130136,250 -rdflex,interacted cutoff and score,Global linear,Stacked,0.95,0.968,12.662531896478109,2.6159768660130136,250 -rdflex,interacted cutoff and score,LGBM,Global linear,0.9,0.936,2.1565580135903732,0.5032955415920486,250 -rdflex,interacted cutoff and score,LGBM,Global linear,0.95,0.98,2.569697368781784,0.5032955415920486,250 -rdflex,interacted cutoff and score,LGBM,LGBM,0.9,0.944,2.2329626116556094,0.5196735971141546,250 -rdflex,interacted cutoff and score,LGBM,LGBM,0.95,0.98,2.6607390627096894,0.5196735971141546,250 -rdflex,interacted cutoff and score,LGBM,Linear,0.9,0.94,2.188972984996604,0.5079568034409114,250 -rdflex,interacted cutoff and score,LGBM,Linear,0.95,0.988,2.608322189540976,0.5079568034409114,250 -rdflex,interacted cutoff and score,LGBM,Stacked,0.9,0.94,2.1435376000141364,0.5369566034254465,250 -rdflex,interacted cutoff and score,LGBM,Stacked,0.95,0.988,2.55418258907428,0.5369566034254465,250 -rdflex,interacted cutoff and score,Linear,Global linear,0.9,0.92,10.689912181088147,2.611324508403497,250 -rdflex,interacted cutoff and score,Linear,Global linear,0.95,0.968,12.737816015678167,2.611324508403497,250 -rdflex,interacted cutoff and score,Linear,LGBM,0.9,0.936,10.997728797394373,2.7029090494109984,250 -rdflex,interacted cutoff and score,Linear,LGBM,0.95,0.976,13.104602136897563,2.7029090494109984,250 -rdflex,interacted cutoff and score,Linear,Linear,0.9,0.936,10.779112620544133,2.610958342807748,250 -rdflex,interacted cutoff and score,Linear,Linear,0.95,0.968,12.844104895049702,2.610958342807748,250 -rdflex,interacted cutoff and score,Linear,Stacked,0.9,0.92,10.737943290197988,2.6468998393072347,250 -rdflex,interacted cutoff and score,Linear,Stacked,0.95,0.968,12.795048612214599,2.6468998393072347,250 -rdflex,interacted cutoff and score,Stacked,Global linear,0.9,0.932,2.233890329611388,0.5262031584833204,250 -rdflex,interacted cutoff and score,Stacked,Global linear,0.95,0.976,2.661844506836355,0.5262031584833204,250 -rdflex,interacted cutoff and score,Stacked,LGBM,0.9,0.936,2.20946398801941,0.5095892633671805,250 -rdflex,interacted cutoff and score,Stacked,LGBM,0.95,0.98,2.632738725622813,0.5095892633671805,250 -rdflex,interacted cutoff and score,Stacked,Linear,0.9,0.928,2.2332457187032886,0.49902647466338274,250 -rdflex,interacted cutoff and score,Stacked,Linear,0.95,0.968,2.661076405563868,0.49902647466338274,250 -rdflex,interacted cutoff and score,Stacked,Stacked,0.9,0.948,2.181302390767035,0.4919155381101698,250 -rdflex,interacted cutoff and score,Stacked,Stacked,0.95,0.984,2.599182112768406,0.4919155381101698,250 -rdrobust,cutoff,linear,linear,0.9,0.928,10.396400783944904,2.560336967798922,250 -rdrobust,cutoff,linear,linear,0.95,0.964,12.388075614449283,2.560336967798922,250 +Method,fs_specification,Learner g,Learner m,level,Coverage,CI Length,Bias,repetition +RDFlex,cutoff,Global Linear,Global Logistic,0.9,0.8943333333333334,9.45138989718295,2.373538922365825,1000 +RDFlex,cutoff,Global Linear,Global Logistic,0.95,0.9466666666666667,11.26202568957878,2.373538922365825,1000 +RDFlex,cutoff,LGBM Regr.,LGBM Clas.,0.9,0.9106666666666666,2.098970644289801,0.5229709395139468,1000 +RDFlex,cutoff,LGBM Regr.,LGBM Clas.,0.95,0.9606666666666667,2.5010777858935986,0.5229709395139468,1000 +RDFlex,cutoff,Linear,Logistic,0.9,0.898,9.475050602811462,2.38122109508206,1000 +RDFlex,cutoff,Linear,Logistic,0.95,0.9516666666666667,11.290219159271665,2.38122109508206,1000 +RDFlex,cutoff,Stacked Regr.,Stacked Clas.,0.9,0.9143333333333333,2.006664818935384,0.4926541115149405,1000 +RDFlex,cutoff,Stacked Regr.,Stacked Clas.,0.95,0.9643333333333334,2.3910886109946707,0.4926541115149405,1000 +RDFlex,cutoff and score,Global Linear,Global Logistic,0.9,0.896,9.45192675987028,2.3708700911814633,1000 +RDFlex,cutoff and score,Global Linear,Global Logistic,0.95,0.9483333333333334,11.262665400927295,2.3708700911814633,1000 +RDFlex,cutoff and score,LGBM Regr.,LGBM Clas.,0.9,0.9206666666666666,2.137979587602868,0.5310240118035273,1000 +RDFlex,cutoff and score,LGBM Regr.,LGBM Clas.,0.95,0.9706666666666667,2.547559808801788,0.5310240118035273,1000 +RDFlex,cutoff and score,Linear,Logistic,0.9,0.8993333333333333,9.431787596286874,2.370891486466498,1000 +RDFlex,cutoff and score,Linear,Logistic,0.95,0.9506666666666667,11.238668107395837,2.370891486466498,1000 +RDFlex,cutoff and score,Stacked Regr.,Stacked Clas.,0.9,0.9206666666666666,2.021432595475123,0.4848163059226252,1000 +RDFlex,cutoff and score,Stacked Regr.,Stacked Clas.,0.95,0.968,2.408685502095107,0.4848163059226252,1000 +RDFlex,interacted cutoff and score,Global Linear,Global Logistic,0.9,0.8986666666666666,9.417430458001911,2.3532260082168035,1000 +RDFlex,interacted cutoff and score,Global Linear,Global Logistic,0.95,0.9506666666666667,11.221560521955702,2.3532260082168035,1000 +RDFlex,interacted cutoff and score,LGBM Regr.,LGBM Clas.,0.9,0.9203333333333333,2.1443333037007934,0.5343555930536144,1000 +RDFlex,interacted cutoff and score,LGBM Regr.,LGBM Clas.,0.95,0.9696666666666667,2.555130728496937,0.5343555930536144,1000 +RDFlex,interacted cutoff and score,Linear,Logistic,0.9,0.8983333333333333,9.463034507707663,2.36467701566644,1000 +RDFlex,interacted cutoff and score,Linear,Logistic,0.95,0.9486666666666667,11.275901098836155,2.36467701566644,1000 +RDFlex,interacted cutoff and score,Stacked Regr.,Stacked Clas.,0.9,0.9233333333333333,2.0607292105190114,0.5038394189514153,1000 +RDFlex,interacted cutoff and score,Stacked Regr.,Stacked Clas.,0.95,0.9763333333333334,2.455510307012918,0.5038394189514153,1000 +rdrobust,cutoff,Linear,Logistic,0.9,0.925,10.188022696252679,2.4726265121800406,1000 +rdrobust,cutoff,Linear,Logistic,0.95,0.97,12.13977780827851,2.4726265121800406,1000 diff --git a/results/rdd/rdd_fuzzy_metadata.csv b/results/rdd/rdd_fuzzy_metadata.csv new file mode 100644 index 0000000..0c28df1 --- /dev/null +++ b/results/rdd/rdd_fuzzy_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.10.0,RDDCoverageSimulation,2025-06-03 10:17,24.518820464611053,3.12.9,scripts/rdd/rdd_fuzzy_config.yml diff --git a/results/rdd/rdd_sharp_config.yml b/results/rdd/rdd_sharp_config.yml new file mode 100644 index 0000000..57d0a43 --- /dev/null +++ b/results/rdd/rdd_sharp_config.yml @@ -0,0 +1,41 @@ +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + n_obs: + - 1000 + fuzzy: + - false + cutoff: + - 0.0 +learner_definitions: + lgbmr: &id001 + name: LGBM Regr. + params: + n_estimators: 100 + learning_rate: 0.05 + global_linear: &id002 + name: Global Linear + local_linear: &id003 + name: Linear + stacked_reg: &id004 + name: Stacked Regr. + params: + n_estimators: 100 + learning_rate: 0.05 +dml_parameters: + fs_specification: + - cutoff + - cutoff and score + - interacted cutoff and score + learners: + - ml_g: *id001 + - ml_g: *id002 + - ml_g: *id003 + - ml_g: *id004 +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/rdd/rdd_sharp_coverage.csv b/results/rdd/rdd_sharp_coverage.csv index 558f06e..f9b942e 100644 --- a/results/rdd/rdd_sharp_coverage.csv +++ b/results/rdd/rdd_sharp_coverage.csv @@ -1,27 +1,27 @@ -Method,fs specification,Learner g,level,Coverage,CI Length,Bias,repetition -rdflex,cutoff,Global linear,0.9,0.874,2.2011088216647567,0.5534513823633507,500 -rdflex,cutoff,Global linear,0.95,0.922,2.6227829308507813,0.5534513823633507,500 -rdflex,cutoff,LGBM,0.9,0.914,0.5720094363002372,0.1386055523323647,500 -rdflex,cutoff,LGBM,0.95,0.96,0.681591283014874,0.1386055523323647,500 -rdflex,cutoff,Linear,0.9,0.874,2.2135060296971463,0.558126057195101,500 -rdflex,cutoff,Linear,0.95,0.918,2.6375551153504855,0.558126057195101,500 -rdflex,cutoff,Stacked,0.9,0.9,0.5590842827226126,0.12977180379581715,500 -rdflex,cutoff,Stacked,0.95,0.964,0.6661900125968221,0.12977180379581715,500 -rdflex,cutoff and score,Global linear,0.9,0.876,2.2005650015589864,0.5551812535814453,500 -rdflex,cutoff and score,Global linear,0.95,0.922,2.622134929226859,0.5551812535814453,500 -rdflex,cutoff and score,LGBM,0.9,0.902,0.5984294788353389,0.14476299319184177,500 -rdflex,cutoff and score,LGBM,0.95,0.95,0.7130727054286046,0.14476299319184177,500 -rdflex,cutoff and score,Linear,0.9,0.87,2.212887444081595,0.5582781571131341,500 -rdflex,cutoff and score,Linear,0.95,0.922,2.63681802512679,0.5582781571131341,500 -rdflex,cutoff and score,Stacked,0.9,0.88,0.5820101876494705,0.14231610717047102,500 -rdflex,cutoff and score,Stacked,0.95,0.956,0.6935079132497274,0.14231610717047102,500 -rdflex,interacted cutoff and score,Global linear,0.9,0.878,2.202268208292408,0.5546663773642981,500 -rdflex,interacted cutoff and score,Global linear,0.95,0.926,2.6241644252264025,0.5546663773642981,500 -rdflex,interacted cutoff and score,LGBM,0.9,0.886,0.6002731299225453,0.151487055237104,500 -rdflex,interacted cutoff and score,LGBM,0.95,0.948,0.7152695511975984,0.151487055237104,500 -rdflex,interacted cutoff and score,Linear,0.9,0.88,2.2252512728137637,0.5541600195953326,500 -rdflex,interacted cutoff and score,Linear,0.95,0.916,2.651550435737079,0.5541600195953326,500 -rdflex,interacted cutoff and score,Stacked,0.9,0.904,0.5793073094025809,0.14171607962579613,500 -rdflex,interacted cutoff and score,Stacked,0.95,0.962,0.6902872351713266,0.14171607962579613,500 -rdrobust,cutoff,linear,0.9,0.874,2.1797037623552287,0.555486091306879,500 -rdrobust,cutoff,linear,0.95,0.916,2.597277229525019,0.555486091306879,500 +Method,fs_specification,Learner g,Learner m,level,Coverage,CI Length,Bias,repetition +RDFlex,cutoff,Global Linear,N/A,0.9,0.8783333333333334,1.9718196650713942,0.5093136469698075,1000 +RDFlex,cutoff,Global Linear,N/A,0.95,0.9376666666666666,2.3495680492315225,0.5093136469698075,1000 +RDFlex,cutoff,LGBM Regr.,N/A,0.9,0.8753333333333334,0.5757111703216161,0.15519089437437322,1000 +RDFlex,cutoff,LGBM Regr.,N/A,0.95,0.9316666666666666,0.6860021711591865,0.15519089437437322,1000 +RDFlex,cutoff,Linear,N/A,0.9,0.8823333333333334,1.9835872651996145,0.5100383308045482,1000 +RDFlex,cutoff,Linear,N/A,0.95,0.942,2.363590009640563,0.5100383308045482,1000 +RDFlex,cutoff,Stacked Regr.,N/A,0.9,0.882,0.5590578781507813,0.14332464663035524,1000 +RDFlex,cutoff,Stacked Regr.,N/A,0.95,0.9366666666666666,0.6661585496088891,0.14332464663035524,1000 +RDFlex,cutoff and score,Global Linear,N/A,0.9,0.882,1.971849810438757,0.5080389655599933,1000 +RDFlex,cutoff and score,Global Linear,N/A,0.95,0.9383333333333334,2.34960396965226,0.5080389655599933,1000 +RDFlex,cutoff and score,LGBM Regr.,N/A,0.9,0.8846666666666666,0.6017477258893029,0.15725459328740565,1000 +RDFlex,cutoff and score,LGBM Regr.,N/A,0.95,0.9423333333333334,0.7170266406669815,0.15725459328740565,1000 +RDFlex,cutoff and score,Linear,N/A,0.9,0.8826666666666666,1.9835459135793096,0.5104954020103588,1000 +RDFlex,cutoff and score,Linear,N/A,0.95,0.9403333333333334,2.363540736145845,0.5104954020103588,1000 +RDFlex,cutoff and score,Stacked Regr.,N/A,0.9,0.8973333333333333,0.5804002753516253,0.14754701246061155,1000 +RDFlex,cutoff and score,Stacked Regr.,N/A,0.95,0.9413333333333334,0.6915895844268198,0.14754701246061155,1000 +RDFlex,interacted cutoff and score,Global Linear,N/A,0.9,0.882,1.9748055181211774,0.508213715385231,1000 +RDFlex,interacted cutoff and score,Global Linear,N/A,0.95,0.938,2.3531259125847197,0.508213715385231,1000 +RDFlex,interacted cutoff and score,LGBM Regr.,N/A,0.9,0.889,0.6013814494909888,0.15466636196782954,1000 +RDFlex,interacted cutoff and score,LGBM Regr.,N/A,0.95,0.9433333333333334,0.7165901954190156,0.15466636196782954,1000 +RDFlex,interacted cutoff and score,Linear,N/A,0.9,0.88,1.992308168049787,0.5113923914828905,1000 +RDFlex,interacted cutoff and score,Linear,N/A,0.95,0.938,2.373981606326701,0.5113923914828905,1000 +RDFlex,interacted cutoff and score,Stacked Regr.,N/A,0.9,0.8756666666666666,0.5801425055800902,0.14962735197253088,1000 +RDFlex,interacted cutoff and score,Stacked Regr.,N/A,0.95,0.929,0.6912824327993231,0.14962735197253088,1000 +rdrobust,cutoff,Linear,Logistic,0.9,0.896,2.178474435204289,0.5326223192522394,1000 +rdrobust,cutoff,Linear,Logistic,0.95,0.954,2.595812395875642,0.5326223192522394,1000 diff --git a/results/rdd/rdd_sharp_metadata.csv b/results/rdd/rdd_sharp_metadata.csv new file mode 100644 index 0000000..375d001 --- /dev/null +++ b/results/rdd/rdd_sharp_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,RDDCoverageSimulation,2025-06-02 13:17,65.69356084664663,3.12.3,scripts/rdd/rdd_sharp_config.yml diff --git a/scripts/did/did_pa_multi_config.yml b/scripts/did/did_pa_multi_config.yml index 67eead1..c89ef8d 100644 --- a/scripts/did/did_pa_multi_config.yml +++ b/scripts/did/did_pa_multi_config.yml @@ -10,13 +10,33 @@ dgp_parameters: DGP: [1, 4, 6] # Different DGP specifications n_obs: [2000] # Sample size for each simulation (has to be a list) +# Define reusable learner configurations +learner_definitions: + linear: &linear + name: "Linear" + + logistic: &logistic + name: "Logistic" + + lgbmr: &lgbmr + name: "LGBM Regr." + params: + n_estimators: 500 + learning_rate: 0.02 + + lgbmc: &lgbmc + name: "LGBM Clas." + params: + n_estimators: 500 + learning_rate: 0.02 + dml_parameters: # ML methods for ml_g and ml_m learners: - - ml_g: ["Linear"] - ml_m: ["Linear"] - - ml_g: ["LGBM"] - ml_m: ["LGBM"] + - ml_g: *linear + ml_m: *logistic + - ml_g: *lgbmr + ml_m: *lgbmc score: - observational # Standard DML score diff --git a/scripts/rdd/rdd_fuzzy.py b/scripts/rdd/rdd_fuzzy.py new file mode 100644 index 0000000..3fec90e --- /dev/null +++ b/scripts/rdd/rdd_fuzzy.py @@ -0,0 +1,13 @@ +from montecover.rdd import RDDCoverageSimulation + +# Create and run simulation with config file +sim = RDDCoverageSimulation( + config_file="scripts/rdd/rdd_fuzzy_config.yml", + log_level="INFO", + log_file="logs/rdd/rdd_fuzzy_sim.log", +) +sim.run_simulation() +sim.save_results(output_path="results/rdd/", file_prefix="rdd_fuzzy") + +# Save config file for reproducibility +sim.save_config("results/rdd/rdd_fuzzy_config.yml") diff --git a/scripts/rdd/rdd_fuzzy_config.yml b/scripts/rdd/rdd_fuzzy_config.yml new file mode 100644 index 0000000..2e9cdc7 --- /dev/null +++ b/scripts/rdd/rdd_fuzzy_config.yml @@ -0,0 +1,70 @@ +# Simulation parameters for fuzzy RDD Coverage + +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 # 5.5 hours in seconds + random_seed: 42 + n_jobs: -2 + +dgp_parameters: + n_obs: [2000] # Sample size + fuzzy: [True] + cutoff: [0.0] + +# Define reusable learner configurations +learner_definitions: + lgbmr: &lgbmr + name: "LGBM Regr." + params: + n_estimators: 200 + learning_rate: 0.02 + max_depth: 5 + + lgbmc: &lgbmc + name: "LGBM Clas." + params: + n_estimators: 200 + learning_rate: 0.02 + max_depth: 5 + + global_linear: &global_linear + name: "Global Linear" + + global_logistic: &global_logistic + name: "Global Logistic" + + local_linear: &local_linear + name: "Linear" + + local_logistic: &local_logistic + name: "Logistic" + + stacked_reg: &stacked_reg + name: "Stacked Regr." + params: + n_estimators: 200 + learning_rate: 0.02 + max_depth: 5 + + stacked_cls: &stacked_cls + name: "Stacked Clas." + params: + n_estimators: 200 + learning_rate: 0.02 + max_depth: 5 + +dml_parameters: + fs_specification: ["cutoff", "cutoff and score", "interacted cutoff and score"] + + learners: + - ml_g: *lgbmr + ml_m: *lgbmc + - ml_g: *global_linear + ml_m: *global_logistic + - ml_g: *local_linear + ml_m: *local_logistic + - ml_g: *stacked_reg + ml_m: *stacked_cls + +confidence_parameters: + level: [0.95, 0.90] # Confidence levels diff --git a/scripts/rdd/rdd_sharp.py b/scripts/rdd/rdd_sharp.py new file mode 100644 index 0000000..3520881 --- /dev/null +++ b/scripts/rdd/rdd_sharp.py @@ -0,0 +1,13 @@ +from montecover.rdd import RDDCoverageSimulation + +# Create and run simulation with config file +sim = RDDCoverageSimulation( + config_file="scripts/rdd/rdd_sharp_config.yml", + log_level="INFO", + log_file="logs/rdd/rdd_sharp_sim.log", +) +sim.run_simulation() +sim.save_results(output_path="results/rdd/", file_prefix="rdd_sharp") + +# Save config file for reproducibility +sim.save_config("results/rdd/rdd_sharp_config.yml") diff --git a/scripts/rdd/rdd_sharp_config.yml b/scripts/rdd/rdd_sharp_config.yml new file mode 100644 index 0000000..560c913 --- /dev/null +++ b/scripts/rdd/rdd_sharp_config.yml @@ -0,0 +1,45 @@ +# Simulation parameters for sharp RDD Coverage + +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 # 5.5 hours in seconds + random_seed: 42 + n_jobs: -2 + +dgp_parameters: + n_obs: [1000] # Sample size + fuzzy: [False] + cutoff: [0.0] + +# Define reusable learner configurations +learner_definitions: + lgbmr: &lgbmr + name: "LGBM Regr." + params: + n_estimators: 100 + learning_rate: 0.05 + + global_linear: &global_linear + name: "Global Linear" + + local_linear: &local_linear + name: "Linear" + + stacked_reg: &stacked_reg + name: "Stacked Regr." + params: + n_estimators: 100 + learning_rate: 0.05 + +dml_parameters: + fs_specification: ["cutoff", "cutoff and score", "interacted cutoff and score"] + + learners: + - ml_g: *lgbmr + - ml_g: *global_linear + - ml_g: *local_linear + - ml_g: *stacked_reg + + +confidence_parameters: + level: [0.95, 0.90] # Confidence levels