diff --git a/.github/workflows/apo_sim.yml b/.github/workflows/apo_sim.yml index 1284d2a..1cbdc3f 100644 --- a/.github/workflows/apo_sim.yml +++ b/.github/workflows/apo_sim.yml @@ -59,7 +59,8 @@ jobs: - name: Install DoubleML from correct branch run: | - pip install git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }} + pip uninstall -y doubleml + pip install "doubleml @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}" - name: Set up Git configuration run: | diff --git a/.github/workflows/did_sim.yml b/.github/workflows/did_sim.yml index c1ca6df..75c3ffc 100644 --- a/.github/workflows/did_sim.yml +++ b/.github/workflows/did_sim.yml @@ -60,7 +60,8 @@ jobs: - name: Install DoubleML from correct branch run: | - pip install git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }} + pip uninstall -y doubleml + pip install "doubleml @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}" - name: Set up Git configuration run: | diff --git a/.github/workflows/iivm_sim.yml b/.github/workflows/iivm_sim.yml index 87aacf0..327962c 100644 --- a/.github/workflows/iivm_sim.yml +++ b/.github/workflows/iivm_sim.yml @@ -59,7 +59,8 @@ jobs: - name: Install DoubleML from correct branch run: | - pip install git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }} + pip uninstall -y doubleml + pip install "doubleml @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}" - name: Set up Git configuration run: | diff --git a/.github/workflows/irm_sim.yml b/.github/workflows/irm_sim.yml index 813d942..63f5f55 100644 --- a/.github/workflows/irm_sim.yml +++ b/.github/workflows/irm_sim.yml @@ -64,7 +64,8 @@ jobs: - name: Install DoubleML from correct branch run: | - pip install git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }} + pip uninstall -y doubleml + pip install "doubleml @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}" - name: Set up Git configuration run: | diff --git a/.github/workflows/pliv_sim.yml b/.github/workflows/pliv_sim.yml index eeb30a1..9c6921e 100644 --- a/.github/workflows/pliv_sim.yml +++ b/.github/workflows/pliv_sim.yml @@ -59,7 +59,8 @@ jobs: - name: Install DoubleML from correct branch run: | - pip install git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }} + pip uninstall -y doubleml + pip install "doubleml @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}" - name: Set up Git configuration run: | diff --git a/.github/workflows/plr_sim.yml b/.github/workflows/plr_sim.yml index 5c02113..7d95d3c 100644 --- a/.github/workflows/plr_sim.yml +++ b/.github/workflows/plr_sim.yml @@ -62,6 +62,7 @@ jobs: - name: Install DoubleML from correct branch run: | + pip uninstall -y doubleml pip install git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }} - name: Set up Git configuration diff --git a/.github/workflows/quant_sim.yml b/.github/workflows/quant_sim.yml index 931b1d2..af1ef7f 100644 --- a/.github/workflows/quant_sim.yml +++ b/.github/workflows/quant_sim.yml @@ -61,7 +61,8 @@ jobs: - name: Install DoubleML from correct branch run: | - pip install git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }} + pip uninstall -y doubleml + pip install "doubleml @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}" - name: Set up Git configuration run: | diff --git a/.github/workflows/rdd_sim.yml b/.github/workflows/rdd_sim.yml index 3d78822..533a6c6 100644 --- a/.github/workflows/rdd_sim.yml +++ b/.github/workflows/rdd_sim.yml @@ -12,7 +12,7 @@ on: types: [run-rdd-scripts] jobs: - run-apo-scripts: + run-rdd-scripts: runs-on: ubuntu-latest strategy: matrix: @@ -60,7 +60,14 @@ jobs: - name: Install DoubleML from correct branch run: | - pip install git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }} + pip uninstall -y doubleml + pip install "doubleml[rdd] @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}" + + - name: Install RDFlex from main branch + run: | + pip uninstall -y doubleml + pip install git+https://github.com/DoubleML/doubleml-rdflex.git@main + pip install rdrobust - name: Set up Git configuration run: | @@ -72,7 +79,7 @@ jobs: - name: Commit any existing changes run: | - git add results/irm + git add results/rdd git commit -m "Update results from script: ${{ matrix.script }}" || echo "No changed results to commit" - name: Wait random time diff --git a/.github/workflows/sim.yml b/.github/workflows/sim.yml index 92f8cda..fce7936 100644 --- a/.github/workflows/sim.yml +++ b/.github/workflows/sim.yml @@ -45,3 +45,14 @@ jobs: echo "HTTP Status: $RESPONSE" cat response.txt + + - name: Trigger RDD Scripts Workflows + run: | + RESPONSE=$(curl -s -o response.txt -w "%{http_code}" -X POST \ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.everest-preview+json" \ + https://api.github.com/repos/${{ github.repository }}/dispatches \ + -d '{"event_type": "run-rdd-scripts", "client_payload": { "doubleml_py_branch": "${{ github.event.inputs.doubleml-py-branch }}", "target_branch": "${{ github.ref_name }}"}}') + + echo "HTTP Status: $RESPONSE" + cat response.txt diff --git a/.github/workflows/ssm_sim.yml b/.github/workflows/ssm_sim.yml index cc2bdb7..e5a7474 100644 --- a/.github/workflows/ssm_sim.yml +++ b/.github/workflows/ssm_sim.yml @@ -60,7 +60,8 @@ jobs: - name: Install DoubleML from correct branch run: | - pip install git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }} + pip uninstall -y doubleml + pip install "doubleml @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}" - name: Set up Git configuration run: | diff --git a/doc/_website.yml b/doc/_website.yml index 7bd5b90..6dd3260 100644 --- a/doc/_website.yml +++ b/doc/_website.yml @@ -1,7 +1,7 @@ website: title: "DoubleML Coverage" favicon: _static/favicon.ico - search: + search: location: sidebar sidebar: style: "docked" @@ -18,20 +18,23 @@ website: - irm/apo.qmd - irm/qte.qmd - text: "PLM" - menu: + menu: - plm/plr.qmd - plm/plr_gate.qmd - plm/plr_cate.qmd - plm/pliv.qmd - text: "DID" - menu: + menu: - did/did_pa.qmd - did/did_cs.qmd - text: "SSM" - menu: + menu: - ssm/ssm_mar.qmd - ssm/ssm_nonignorable.qmd - right: + - text: "RDD" + menu: + - rdd/rdd.qmd + right: - icon: book href: https://docs.doubleml.org/stable/index.html aria-label: Documentation @@ -43,9 +46,8 @@ website: href: https://github.com/DoubleML/doubleml-coverage aria-label: GitHub - format: html: - theme: cosmo + theme: cosmo css: styles.css toc: true diff --git a/doc/rdd/rdd.qmd b/doc/rdd/rdd.qmd new file mode 100644 index 0000000..71cd93c --- /dev/null +++ b/doc/rdd/rdd.qmd @@ -0,0 +1,149 @@ +--- +title: "Flexible covariate adjustments in RDD" + +jupyter: python3 +--- + + +```{python} +#| echo: false + +import numpy as np +import pandas as pd +from itables import init_notebook_mode, show, options + +init_notebook_mode(all_interactive=True) + +def highlight_range(s, level=0.95, dist=0.05, props=''): + color_grid = np.where((s >= level-dist) & + (s <= level+dist), props, '') + return color_grid + + +def color_coverage(df, level): + # color coverage column order is important + styled_df = df.apply( + highlight_range, + level=level, + dist=1.0, + props='color:black;background-color:red', + subset=["Coverage"]) + styled_df = styled_df.apply( + highlight_range, + level=level, + dist=0.1, + props='color:black;background-color:yellow', + subset=["Coverage"]) + styled_df = styled_df.apply( + highlight_range, + level=level, + dist=0.05, + props='color:white;background-color:darkgreen', + subset=["Coverage"]) + + # set all coverage values to bold + styled_df = styled_df.set_properties( + **{'font-weight': 'bold'}, + subset=["Coverage"]) + return styled_df + + +def make_pretty(df, level, n_rep): + styled_df = df.style.hide(axis="index") + # Format only float columns + float_cols = df.select_dtypes(include=['float']).columns + styled_df = styled_df.format({col: "{:.3f}" for col in float_cols}) + + # color coverage column order is important + styled_df = color_coverage(styled_df, level) + caption = f"Coverage for {level*100}%-Confidence Interval over {n_rep} Repetitions" + + return show(styled_df, caption=caption) +``` + +## Sharp Design + +The simulations are based on the [make_simple_rdd_data](https://docs.doubleml.org/stable/api/generated/doubleml.rdd.datasets.make_simple_rdd_data.html#doubleml.rdd.datasets.make_simple_rdd_data)-DGP with $1000$ observations. The simulation considers data under a [sharp regression discontinuity design](https://docs.doubleml.org/stable/guide/models.html#sharp-regression-discontinuity-design). + +::: {.callout-note title="Metadata" collapse="true"} + +```{python} +#| echo: false +metadata_file = '../../results/rdd/rdd_sharp_coverage_metadata.csv' +metadata_df = pd.read_csv(metadata_file) +print(metadata_df.T.to_string(header=False)) +``` + +::: + +```{python} +# | echo: false + +# set up data and rename columns +df = pd.read_csv("../../results/rdd/rdd_sharp_coverage.csv", index_col=None) + +assert df["repetition"].nunique() == 1 +n_rep = df["repetition"].unique()[0] + +display_columns = ["Method", "Learner g", "fs specification", "Bias", "CI Length", "Coverage"] +``` + +```{python} +#| echo: false +level = 0.95 + +df_ate_95 = df[(df['level'] == level)][display_columns] +make_pretty(df_ate_95, level, n_rep) +``` + +```{python} +#| echo: false +level = 0.9 + +df_ate_9 = df[(df['level'] == level)][display_columns] +make_pretty(df_ate_9, level, n_rep) +``` + + +## Fuzzy Design + +The simulations are based on the [make_simple_rdd_data](https://docs.doubleml.org/stable/api/generated/doubleml.rdd.datasets.make_simple_rdd_data.html#doubleml.rdd.datasets.make_simple_rdd_data)-DGP with $2000$ observations. The simulation considers data under a [fuzzy regression discontinuity design](https://docs.doubleml.org/stable/guide/models.html#fuzzy-regression-discontinuity-design). + +::: {.callout-note title="Metadata" collapse="true"} + +```{python} +#| echo: false +metadata_file = '../../results/rdd/rdd_fuzzy_coverage_metadata.csv' +metadata_df = pd.read_csv(metadata_file) +print(metadata_df.T.to_string(header=False)) +``` + +::: + +```{python} +# | echo: false + +# set up data and rename columns +df = pd.read_csv("../../results/rdd/rdd_fuzzy_coverage.csv", index_col=None) + +assert df["repetition"].nunique() == 1 +n_rep = df["repetition"].unique()[0] + +display_columns = ["Method", "Learner g", "Learner m", "fs specification", "Bias", "CI Length", "Coverage"] +``` + +```{python} +#| echo: false +level = 0.95 + +df_ate_95 = df[(df['level'] == level)][display_columns] +make_pretty(df_ate_95, level, n_rep) +``` + +```{python} +#| echo: false +level = 0.9 + +df_ate_9 = df[(df['level'] == level)][display_columns] +make_pretty(df_ate_9, level, n_rep) +``` \ No newline at end of file diff --git a/doc/ssm/ssm_mar.qmd b/doc/ssm/ssm_mar.qmd index 1a7e033..0d52537 100644 --- a/doc/ssm/ssm_mar.qmd +++ b/doc/ssm/ssm_mar.qmd @@ -62,7 +62,7 @@ def make_pretty(df, level, n_rep): ## ATE Coverage -The simulations are based on the the [make_ssm_data](https://docs.doubleml.org/stable/api/generated/doubleml.datasets.make_ssm_data.html)-DGP with $500$ observations. The simulation considers data under [missingness at random](https://docs.doubleml.org/stable/guide/models.html#missingness-at-random). +The simulations are based on the [make_ssm_data](https://docs.doubleml.org/stable/api/generated/doubleml.datasets.make_ssm_data.html)-DGP with $500$ observations. The simulation considers data under [missingness at random](https://docs.doubleml.org/stable/guide/models.html#missingness-at-random). ::: {.callout-note title="Metadata" collapse="true"} diff --git a/doc/ssm/ssm_nonignorable.qmd b/doc/ssm/ssm_nonignorable.qmd index 44c55d1..fee4929 100644 --- a/doc/ssm/ssm_nonignorable.qmd +++ b/doc/ssm/ssm_nonignorable.qmd @@ -62,7 +62,7 @@ def make_pretty(df, level, n_rep): ## ATE Coverage -The simulations are based on the the [make_ssm_data](https://docs.doubleml.org/stable/api/generated/doubleml.datasets.make_ssm_data.html)-DGP with $500$ observations. The simulation considers data with [nonignorable nonresponse](https://docs.doubleml.org/stable/guide/models.html#nonignorable-nonresponse). +The simulations are based on the [make_ssm_data](https://docs.doubleml.org/stable/api/generated/doubleml.datasets.make_ssm_data.html)-DGP with $500$ observations. The simulation considers data with [nonignorable nonresponse](https://docs.doubleml.org/stable/guide/models.html#nonignorable-nonresponse). ::: {.callout-note title="Metadata" collapse="true"} diff --git a/requirements.txt b/requirements.txt index a58fef6..2accaab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -doubleml +doubleml[rdd] joblib numpy pandas diff --git a/results/did/did_cs_atte_coverage_metadata.csv b/results/did/did_cs_atte_coverage_metadata.csv index a29c07a..d6308e5 100644 --- a/results/did/did_cs_atte_coverage_metadata.csv +++ b/results/did/did_cs_atte_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.9.0,did_cs_atte_coverage.py,2024-09-09 12:16:59,17588.891700983047,3.12.5 +0.10.dev0,did_cs_atte_coverage.py,2025-01-08 16:52:06,16598.133431196213,3.12.8 diff --git a/results/did/did_pa_atte_coverage_metadata.csv b/results/did/did_pa_atte_coverage_metadata.csv index b35d01b..65f1786 100644 --- a/results/did/did_pa_atte_coverage_metadata.csv +++ b/results/did/did_pa_atte_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.9.0,did_pa_atte_coverage.py,2024-09-09 11:44:15,15625.949723243713,3.12.5 +0.10.dev0,did_pa_atte_coverage.py,2025-01-08 16:22:26,14811.1988697052,3.12.8 diff --git a/results/irm/cvar_coverage_metadata.csv b/results/irm/cvar_coverage_metadata.csv index a19cb5b..6b80d0c 100644 --- a/results/irm/cvar_coverage_metadata.csv +++ b/results/irm/cvar_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.9.0,cvar_coverage.py,2024-09-09 11:59:26,16546.96124267578,3.12.5 +0.10.dev0,cvar_coverage.py,2025-01-08 16:49:01,16412.1168384552,3.12.8 diff --git a/results/irm/cvar_coverage_pq0.csv b/results/irm/cvar_coverage_pq0.csv index e3a4fa3..a9650d4 100644 --- a/results/irm/cvar_coverage_pq0.csv +++ b/results/irm/cvar_coverage_pq0.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.9,0.8876923076923078,0.5699689541680535,0.13934148600639645,100 -LGBM,LGBM,0.95,0.9461538461538461,0.6791598985897508,0.13934148600639645,100 -LGBM,Logistic Regression,0.9,0.8192307692307692,0.4060118886357097,0.1185635852924032,100 -LGBM,Logistic Regression,0.95,0.89,0.48379300503226824,0.1185635852924032,100 -Linear,LGBM,0.9,0.77,0.5801661718639771,0.17503081399943532,100 -Linear,LGBM,0.95,0.8607692307692308,0.691310632915921,0.17503081399943532,100 -Linear,Logistic Regression,0.9,0.6892307692307692,0.4294636844188598,0.15444826651283644,100 -Linear,Logistic Regression,0.95,0.7776923076923077,0.5117375432906374,0.15444826651283644,100 +LGBM,LGBM,0.9,0.8884615384615384,0.5699689541680535,0.13934954721406115,100 +LGBM,LGBM,0.95,0.9461538461538461,0.6791598985897508,0.13934954721406115,100 +LGBM,Logistic Regression,0.9,0.8192307692307692,0.4060217020758364,0.11848308187809455,100 +LGBM,Logistic Regression,0.95,0.8892307692307692,0.48380469846741503,0.11848308187809455,100 +Linear,LGBM,0.9,0.7692307692307692,0.5801661718639771,0.1751137266106609,100 +Linear,LGBM,0.95,0.8615384615384616,0.691310632915921,0.1751137266106609,100 +Linear,Logistic Regression,0.9,0.6892307692307692,0.42946981193983413,0.1545041280672664,100 +Linear,Logistic Regression,0.95,0.7776923076923077,0.5117448446822186,0.1545041280672664,100 diff --git a/results/irm/cvar_coverage_pq1.csv b/results/irm/cvar_coverage_pq1.csv index 167f61d..a364e69 100644 --- a/results/irm/cvar_coverage_pq1.csv +++ b/results/irm/cvar_coverage_pq1.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.9,0.9323076923076923,0.1908311409902022,0.04347081995558675,100 -LGBM,LGBM,0.95,0.9776923076923077,0.22738932956769187,0.04347081995558675,100 -LGBM,Logistic Regression,0.9,0.9146153846153847,0.1776482729176802,0.04405662937860094,100 -LGBM,Logistic Regression,0.95,0.9646153846153847,0.21168097338831957,0.04405662937860094,100 -Linear,LGBM,0.9,0.9307692307692308,0.21584587388672186,0.04750929738761366,100 -Linear,LGBM,0.95,0.9761538461538461,0.25719622226423833,0.04750929738761366,100 -Linear,Logistic Regression,0.9,0.8853846153846153,0.1934225485324801,0.04921876802543021,100 -Linear,Logistic Regression,0.95,0.943076923076923,0.23047718210904136,0.04921876802543021,100 +LGBM,LGBM,0.9,0.9323076923076923,0.1908311409902022,0.04358145932430658,100 +LGBM,LGBM,0.95,0.9776923076923077,0.22738932956769187,0.04358145932430658,100 +LGBM,Logistic Regression,0.9,0.9130769230769231,0.17764550716816235,0.04414608757632295,100 +LGBM,Logistic Regression,0.95,0.963076923076923,0.21167767779450106,0.04414608757632295,100 +Linear,LGBM,0.9,0.9292307692307692,0.21584587388672186,0.047457925834839765,100 +Linear,LGBM,0.95,0.9769230769230769,0.25719622226423833,0.047457925834839765,100 +Linear,Logistic Regression,0.9,0.8823076923076922,0.19342205011642188,0.0494625033437771,100 +Linear,Logistic Regression,0.95,0.9415384615384617,0.2304765882096771,0.0494625033437771,100 diff --git a/results/irm/cvar_coverage_qte.csv b/results/irm/cvar_coverage_qte.csv index 4e413fe..2dbd870 100644 --- a/results/irm/cvar_coverage_qte.csv +++ b/results/irm/cvar_coverage_qte.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.9007692307692308,0.5818966962059838,0.14385931104084665,0.88,0.7055777033989438,100 -LGBM,LGBM,0.95,0.9492307692307692,0.693372679853793,0.14385931104084665,0.95,0.8113769406399532,100 -LGBM,Logistic Regression,0.9,0.8092307692307692,0.4171534550934474,0.12274374044729776,0.78,0.5062270696539016,100 -LGBM,Logistic Regression,0.95,0.87,0.49706900031277096,0.12274374044729776,0.86,0.5834562695237335,100 -Linear,LGBM,0.9,0.8007692307692308,0.6065907561805581,0.179698169022633,0.8,0.7182897195756133,100 -Linear,LGBM,0.95,0.8653846153846153,0.7227974671960807,0.179698169022633,0.85,0.8301176550726784,100 -Linear,Logistic Regression,0.9,0.7092307692307692,0.454167116768984,0.15248058466727346,0.69,0.5341820696542956,100 -Linear,Logistic Regression,0.95,0.8123076923076923,0.5411734985072126,0.15248058466727346,0.8,0.6185623415096295,100 +LGBM,LGBM,0.9,0.9038461538461539,0.5818966962059838,0.1435944755709475,0.88,0.7055777033989438,100 +LGBM,LGBM,0.95,0.9492307692307692,0.693372679853793,0.1435944755709475,0.94,0.8113769406399532,100 +LGBM,Logistic Regression,0.9,0.81,0.41716340917968886,0.12258144778601979,0.78,0.5062698827872832,100 +LGBM,Logistic Regression,0.95,0.8738461538461538,0.49708086133810014,0.12258144778601979,0.86,0.5835143322993298,100 +Linear,LGBM,0.9,0.8023076923076923,0.6065907561805581,0.1790914579109802,0.8,0.7182897195756133,100 +Linear,LGBM,0.95,0.8653846153846153,0.7227974671960807,0.1790914579109802,0.85,0.8301176550726784,100 +Linear,Logistic Regression,0.9,0.713076923076923,0.4541725983633358,0.15175636482362434,0.71,0.5341780882458428,100 +Linear,Logistic Regression,0.95,0.8146153846153846,0.541180030229751,0.15175636482362434,0.8,0.6185746229300051,100 diff --git a/results/irm/iivm_late_coverage.csv b/results/irm/iivm_late_coverage.csv index 9322f98..a8cd4a5 100644 --- a/results/irm/iivm_late_coverage.csv +++ b/results/irm/iivm_late_coverage.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -Lasso,Logistic Regression,0.9,0.902,0.9293147641314786,0.22417217252344523,1000 -Lasso,Logistic Regression,0.95,0.956,1.1073468411744398,0.22417217252344523,1000 -Lasso,Random Forest,0.9,0.907,0.9551548569821924,0.23030571774269312,1000 -Lasso,Random Forest,0.95,0.965,1.1381372109159926,0.23030571774269312,1000 -Random Forest,Logistic Regression,0.9,0.901,0.9627435648761393,0.2326439704061282,1000 -Random Forest,Logistic Regression,0.95,0.949,1.1471797141014568,0.2326439704061282,1000 -Random Forest,Random Forest,0.9,0.906,0.9916155285633201,0.23508228830266614,1000 -Random Forest,Random Forest,0.95,0.96,1.1815827807711041,0.23508228830266614,1000 +Lasso,Logistic Regression,0.9,0.902,0.9293147641314788,0.2241721725234453,1000 +Lasso,Logistic Regression,0.95,0.956,1.10734684117444,0.2241721725234453,1000 +Lasso,Random Forest,0.9,0.906,0.9553298843030593,0.23080551899825463,1000 +Lasso,Random Forest,0.95,0.96,1.1383457688323866,0.23080551899825463,1000 +Random Forest,Logistic Regression,0.9,0.905,0.9631937131993897,0.23250791933354573,1000 +Random Forest,Logistic Regression,0.95,0.949,1.1477160989121267,0.23250791933354573,1000 +Random Forest,Random Forest,0.9,0.905,0.991691362981023,0.23403696617336237,1000 +Random Forest,Random Forest,0.95,0.962,1.181673143053225,0.23403696617336237,1000 diff --git a/results/irm/iivm_late_coverage_metadata.csv b/results/irm/iivm_late_coverage_metadata.csv index 12b58dc..b18008f 100644 --- a/results/irm/iivm_late_coverage_metadata.csv +++ b/results/irm/iivm_late_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.9.0,iivm_late_coverage.py,2024-09-09 09:00:39,5819.892728328705,3.12.5 +0.10.dev0,iivm_late_coverage.py,2025-01-08 13:50:46,5734.383908510208,3.12.8 diff --git a/results/irm/irm_apo_coverage_apo.csv b/results/irm/irm_apo_coverage_apo.csv index ca4b92b..c4e8851 100644 --- a/results/irm/irm_apo_coverage_apo.csv +++ b/results/irm/irm_apo_coverage_apo.csv @@ -1,25 +1,25 @@ Learner g,Learner m,Treatment Level,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.0,0.9,0.912,8.657690136121921,2.0760855366229958,1000 -LGBM,LGBM,0.0,0.95,0.965,10.316274091547035,2.0760855366229958,1000 -LGBM,LGBM,1.0,0.9,0.915,38.23339285821166,9.180525200256618,1000 -LGBM,LGBM,1.0,0.95,0.967,45.55789754237906,9.180525200256618,1000 -LGBM,LGBM,2.0,0.9,0.89,37.49194764946096,9.610674399165429,1000 -LGBM,LGBM,2.0,0.95,0.952,44.67441108385784,9.610674399165429,1000 -LGBM,Logistic,0.0,0.9,0.904,5.625897101886533,1.3345731610342202,1000 -LGBM,Logistic,0.0,0.95,0.955,6.7036698705295725,1.3345731610342202,1000 -LGBM,Logistic,1.0,0.9,0.924,7.423300143143786,1.697057819214677,1000 -LGBM,Logistic,1.0,0.95,0.968,8.84540769378873,1.697057819214677,1000 -LGBM,Logistic,2.0,0.9,0.917,7.321275660150267,1.660826093332619,1000 -LGBM,Logistic,2.0,0.95,0.971,8.723838024042964,1.660826093332619,1000 -Linear,LGBM,0.0,0.9,0.901,5.498257423071024,1.30423568882223,1000 -Linear,LGBM,0.0,0.95,0.95,6.551577812380716,1.30423568882223,1000 -Linear,LGBM,1.0,0.9,0.949,10.700720020780512,2.1264975614784354,1000 -Linear,LGBM,1.0,0.95,0.982,12.75069435099058,2.1264975614784354,1000 -Linear,LGBM,2.0,0.9,0.93,7.513644049429104,1.6338972741785736,1000 -Linear,LGBM,2.0,0.95,0.965,8.953059097926168,1.6338972741785736,1000 -Linear,Logistic,0.0,0.9,0.902,5.335670092717667,1.2832290098794459,1000 -Linear,Logistic,0.0,0.95,0.954,6.357843058957276,1.2832290098794459,1000 -Linear,Logistic,1.0,0.9,0.905,5.417512107920403,1.2812664745065607,1000 -Linear,Logistic,1.0,0.95,0.956,6.455363835025866,1.2812664745065607,1000 -Linear,Logistic,2.0,0.9,0.908,5.366403391397197,1.2775297528572058,1000 -Linear,Logistic,2.0,0.95,0.958,6.3944640430685675,1.2775297528572058,1000 +LGBM,LGBM,0.0,0.9,0.911,8.657690136121921,2.0760918678352267,1000 +LGBM,LGBM,0.0,0.95,0.963,10.316274091547035,2.0760918678352267,1000 +LGBM,LGBM,1.0,0.9,0.914,38.23339285821166,9.19332007236871,1000 +LGBM,LGBM,1.0,0.95,0.967,45.55789754237906,9.19332007236871,1000 +LGBM,LGBM,2.0,0.9,0.891,37.49194764946096,9.594147407062762,1000 +LGBM,LGBM,2.0,0.95,0.952,44.67441108385784,9.594147407062762,1000 +LGBM,Logistic,0.0,0.9,0.901,5.625897101886533,1.3366474407456235,1000 +LGBM,Logistic,0.0,0.95,0.955,6.7036698705295725,1.3366474407456235,1000 +LGBM,Logistic,1.0,0.9,0.92,7.423300143143785,1.6954254817667072,1000 +LGBM,Logistic,1.0,0.95,0.968,8.84540769378873,1.6954254817667072,1000 +LGBM,Logistic,2.0,0.9,0.918,7.321275660150268,1.660607504960853,1000 +LGBM,Logistic,2.0,0.95,0.971,8.723838024042964,1.660607504960853,1000 +Linear,LGBM,0.0,0.9,0.901,5.498257423071024,1.307400569952483,1000 +Linear,LGBM,0.0,0.95,0.949,6.551577812380716,1.307400569952483,1000 +Linear,LGBM,1.0,0.9,0.951,10.700720020780512,2.1270559074131152,1000 +Linear,LGBM,1.0,0.95,0.982,12.75069435099058,2.1270559074131152,1000 +Linear,LGBM,2.0,0.9,0.932,7.513644049429104,1.634643965398859,1000 +Linear,LGBM,2.0,0.95,0.967,8.953059097926168,1.634643965398859,1000 +Linear,Logistic,0.0,0.9,0.9,5.335670092717667,1.2859337252007816,1000 +Linear,Logistic,0.0,0.95,0.951,6.357843058957276,1.2859337252007816,1000 +Linear,Logistic,1.0,0.9,0.904,5.417512107920403,1.2802845007629777,1000 +Linear,Logistic,1.0,0.95,0.954,6.455363835025866,1.2802845007629777,1000 +Linear,Logistic,2.0,0.9,0.905,5.366403391397197,1.2808554104576877,1000 +Linear,Logistic,2.0,0.95,0.959,6.3944640430685675,1.2808554104576877,1000 diff --git a/results/irm/irm_apo_coverage_apos.csv b/results/irm/irm_apo_coverage_apos.csv index 48a0fba..9450b8b 100644 --- a/results/irm/irm_apo_coverage_apos.csv +++ b/results/irm/irm_apo_coverage_apos.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.906,28.21021843292038,7.120657478889542,0.924,36.1943517986729,1000 -LGBM,LGBM,0.95,0.9586666666666667,33.61454856442563,7.120657478889542,0.974,40.868020474452145,1000 -LGBM,Logistic,0.9,0.918,6.789316986971467,1.5705977914191496,0.925,8.394070597867524,1000 -LGBM,Logistic,0.95,0.962,8.089970168806186,1.5705977914191496,0.961,9.592284619044477,1000 -Linear,LGBM,0.9,0.9276666666666666,7.903418354805325,1.7245563336966274,0.94,9.850445774370156,1000 -Linear,LGBM,0.95,0.9666666666666667,9.41750382912841,1.7245563336966274,0.975,11.234392064837424,1000 -Linear,Logistic,0.9,0.903,5.372532806955153,1.278875142245648,0.906,5.7964894175017925,1000 -Linear,Logistic,0.95,0.9556666666666667,6.4017676921854445,1.278875142245648,0.953,6.8182385960659,1000 +LGBM,LGBM,0.9,0.9063333333333333,28.21021843292038,7.1182934844676025,0.925,36.1943517986729,1000 +LGBM,LGBM,0.95,0.9583333333333334,33.61454856442563,7.1182934844676025,0.973,40.868020474452145,1000 +LGBM,Logistic,0.9,0.9153333333333333,6.789316986971467,1.5714825799469632,0.924,8.394070597867524,1000 +LGBM,Logistic,0.95,0.963,8.089970168806186,1.5714825799469632,0.959,9.592284619044477,1000 +Linear,LGBM,0.9,0.9266666666666666,7.903418354805325,1.726298806364679,0.939,9.850445774370156,1000 +Linear,LGBM,0.95,0.967,9.41750382912841,1.726298806364679,0.974,11.234392064837424,1000 +Linear,Logistic,0.9,0.904,5.372532806955153,1.2803595601237736,0.906,5.7964894175017925,1000 +Linear,Logistic,0.95,0.957,6.4017676921854445,1.2803595601237736,0.953,6.8182385960659,1000 diff --git a/results/irm/irm_apo_coverage_apos_contrast.csv b/results/irm/irm_apo_coverage_apos_contrast.csv index 36dea44..c663a0b 100644 --- a/results/irm/irm_apo_coverage_apos_contrast.csv +++ b/results/irm/irm_apo_coverage_apos_contrast.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.8885,37.87536523730769,9.788503277093351,0.898,44.828925781533165,1000 -LGBM,LGBM,0.95,0.9485,45.13128131893863,9.788503277093351,0.965,51.42387403222799,1000 -LGBM,Logistic,0.9,0.9275,5.725128733165455,1.267381146072165,0.927,6.774304315459576,1000 -LGBM,Logistic,0.95,0.963,6.821911652197591,1.267381146072165,0.962,7.7654094416927215,1000 -Linear,LGBM,0.9,0.958,7.430953406859927,1.5064356327113417,0.975,8.798628447594016,1000 -Linear,LGBM,0.95,0.989,8.85452711998085,1.5064356327113417,0.992,10.090116294778499,1000 -Linear,Logistic,0.9,0.872,1.1425883251377777,0.29482653859949187,0.871,1.3505222299078565,1000 -Linear,Logistic,0.95,0.936,1.3614779635902856,0.29482653859949187,0.92,1.5496710496635275,1000 +LGBM,LGBM,0.9,0.8885,37.87536523730769,9.786356896553793,0.898,44.828925781533165,1000 +LGBM,LGBM,0.95,0.9485,45.13128131893863,9.786356896553793,0.965,51.42387403222799,1000 +LGBM,Logistic,0.9,0.9275,5.725128733165455,1.2679354841429349,0.926,6.774304315459575,1000 +LGBM,Logistic,0.95,0.962,6.821911652197591,1.2679354841429349,0.961,7.7654094416927215,1000 +Linear,LGBM,0.9,0.958,7.430953406859927,1.5066894086042675,0.975,8.798628447594016,1000 +Linear,LGBM,0.95,0.9885,8.85452711998085,1.5066894086042675,0.992,10.090116294778499,1000 +Linear,Logistic,0.9,0.873,1.1425883251377777,0.29743753232068365,0.869,1.3505222299078565,1000 +Linear,Logistic,0.95,0.9335,1.3614779635902856,0.29743753232068365,0.916,1.5496710496635275,1000 diff --git a/results/irm/irm_apo_coverage_metadata.csv b/results/irm/irm_apo_coverage_metadata.csv index d2aff99..5fac53a 100644 --- a/results/irm/irm_apo_coverage_metadata.csv +++ b/results/irm/irm_apo_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.9.0,irm_apo_coverage.py,2024-09-09 10:19:31,10550.167822599411,3.12.5 +0.10.dev0,irm_apo_coverage.py,2025-01-08 15:02:49,10054.695460557938,3.12.8 diff --git a/results/irm/irm_ate_coverage.csv b/results/irm/irm_ate_coverage.csv index ea24817..0de042e 100644 --- a/results/irm/irm_ate_coverage.csv +++ b/results/irm/irm_ate_coverage.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -Lasso,Logistic Regression,0.9,0.875,0.467771360497192,0.12336884150536209,1000 -Lasso,Logistic Regression,0.95,0.935,0.5573839547492131,0.12336884150536209,1000 -Lasso,Random Forest,0.9,0.927,0.6001366139904836,0.14798855067476355,1000 -Lasso,Random Forest,0.95,0.96,0.7151068824313491,0.14798855067476355,1000 -Random Forest,Logistic Regression,0.9,0.796,0.5178418585688922,0.15004402486229945,1000 -Random Forest,Logistic Regression,0.95,0.878,0.6170466331179857,0.15004402486229945,1000 -Random Forest,Random Forest,0.9,0.897,0.6233089219687683,0.150114084240768,1000 -Random Forest,Random Forest,0.95,0.953,0.7427183904293482,0.150114084240768,1000 +Lasso,Logistic Regression,0.9,0.875,0.467771360497192,0.12336884150536219,1000 +Lasso,Logistic Regression,0.95,0.935,0.5573839547492132,0.12336884150536219,1000 +Lasso,Random Forest,0.9,0.906,0.6047145364621751,0.1478860899164366,1000 +Lasso,Random Forest,0.95,0.959,0.7205618135094181,0.1478860899164366,1000 +Random Forest,Logistic Regression,0.9,0.803,0.5176227656141883,0.15019297474976653,1000 +Random Forest,Logistic Regression,0.95,0.883,0.6167855677602845,0.15019297474976653,1000 +Random Forest,Random Forest,0.9,0.899,0.6330777189487165,0.15360554047241584,1000 +Random Forest,Random Forest,0.95,0.959,0.7543586299857805,0.15360554047241584,1000 diff --git a/results/irm/irm_ate_coverage_metadata.csv b/results/irm/irm_ate_coverage_metadata.csv index cd30335..a6fbf57 100644 --- a/results/irm/irm_ate_coverage_metadata.csv +++ b/results/irm/irm_ate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.9.0,irm_ate_coverage.py,2024-09-09 08:23:16,3577.3718523979187,3.12.5 +0.10.dev0,irm_ate_coverage.py,2025-01-08 13:14:33,3543.555382013321,3.12.8 diff --git a/results/irm/irm_ate_sensitivity.csv b/results/irm/irm_ate_sensitivity.csv index 34ac6c7..6e10770 100644 --- a/results/irm/irm_ate_sensitivity.csv +++ b/results/irm/irm_ate_sensitivity.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),repetition LGBM,LGBM,0.9,0.112,0.266748233354866,0.17891290135375168,0.962,1.0,0.12379347892727971,0.05409589192160397,0.04254708028278409,0.32210978560617337,500 LGBM,LGBM,0.95,0.318,0.31785012462427936,0.17891290135375168,0.998,1.0,0.12379347892727971,0.03441021667548556,0.04254708028278409,0.32210978560617337,500 -LGBM,Logistic Regr.,0.9,0.292,0.2577778025822409,0.14922926552528684,1.0,1.0,0.100665719512958,0.03493291437943745,0.029012990398602393,0.2979424530565633,500 -LGBM,Logistic Regr.,0.95,0.548,0.30716119707955875,0.14922926552528684,1.0,1.0,0.100665719512958,0.018697523014548616,0.029012990398602393,0.2979424530565633,500 +LGBM,Logistic Regr.,0.9,0.292,0.2577778025822409,0.14922926552528684,1.0,1.0,0.10066571951295798,0.03493291437943745,0.029012990398602386,0.2979424530565633,500 +LGBM,Logistic Regr.,0.95,0.548,0.30716119707955875,0.14922926552528684,1.0,1.0,0.10066571951295798,0.01869752301454861,0.029012990398602386,0.2979424530565633,500 Linear Reg.,LGBM,0.9,0.122,0.2675665174758639,0.17873104426193565,0.964,1.0,0.12647219547900976,0.05512739569620471,0.04513946154555041,0.31857328180879246,500 Linear Reg.,LGBM,0.95,0.314,0.31882517029399604,0.17873104426193565,0.998,1.0,0.12647219547900976,0.035017588858111126,0.04513946154555041,0.31857328180879246,500 -Linear Reg.,Logistic Regr.,0.9,0.86,0.2592281409673473,0.08970251629543106,1.0,1.0,0.06300567732617765,0.006719868195974335,0.05720312141493261,0.23496869651774063,500 -Linear Reg.,Logistic Regr.,0.95,0.974,0.30888938185760084,0.08970251629543106,1.0,1.0,0.06300567732617765,0.0014945204694376403,0.05720312141493261,0.23496869651774063,500 +Linear Reg.,Logistic Regr.,0.9,0.86,0.2592281409673473,0.08970251629543106,1.0,1.0,0.06300567732617765,0.006719868195974334,0.05720312141493262,0.23496869651774063,500 +Linear Reg.,Logistic Regr.,0.95,0.974,0.30888938185760084,0.08970251629543106,1.0,1.0,0.06300567732617765,0.0014945204694376396,0.05720312141493262,0.23496869651774063,500 diff --git a/results/irm/irm_ate_sensitivity_metadata.csv b/results/irm/irm_ate_sensitivity_metadata.csv index 744233a..c038f24 100644 --- a/results/irm/irm_ate_sensitivity_metadata.csv +++ b/results/irm/irm_ate_sensitivity_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.9.0,irm_ate_sensitivity.py,2024-09-09 09:57:25,9228.398559331894,3.12.5 +0.10.dev0,irm_ate_sensitivity.py,2025-01-08 14:51:20,9351.407655954361,3.12.8 diff --git a/results/irm/irm_atte_coverage.csv b/results/irm/irm_atte_coverage.csv index 7c43c9f..3f486c5 100644 --- a/results/irm/irm_atte_coverage.csv +++ b/results/irm/irm_atte_coverage.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -Lasso,Logistic Regression,0.9,0.888,0.5331759172799809,0.13451159762471473,1000 -Lasso,Logistic Regression,0.95,0.941,0.6353182910443254,0.13451159762471473,1000 -Lasso,Random Forest,0.9,0.893,0.7302574772345878,0.179721833597512,1000 -Lasso,Random Forest,0.95,0.956,0.8701554541807854,0.179721833597512,1000 -Random Forest,Logistic Regression,0.9,0.869,0.5521000550346591,0.15039777848508581,1000 -Random Forest,Logistic Regression,0.95,0.922,0.6578677920028921,0.15039777848508581,1000 -Random Forest,Random Forest,0.9,0.902,0.7520609787305578,0.18190144421857504,1000 -Random Forest,Random Forest,0.95,0.945,0.8961359286550816,0.18190144421857504,1000 +Lasso,Logistic Regression,0.9,0.887,0.5331759172799809,0.13448723008182129,1000 +Lasso,Logistic Regression,0.95,0.942,0.6353182910443254,0.13448723008182129,1000 +Lasso,Random Forest,0.9,0.895,0.7320685495582887,0.18096827990713532,1000 +Lasso,Random Forest,0.95,0.948,0.8723134799586961,0.18096827990713532,1000 +Random Forest,Logistic Regression,0.9,0.871,0.5508997607447915,0.1495507044107987,1000 +Random Forest,Logistic Regression,0.95,0.928,0.6564375531412434,0.1495507044107987,1000 +Random Forest,Random Forest,0.9,0.901,0.7500491227406757,0.18282767194851973,1000 +Random Forest,Random Forest,0.95,0.948,0.8937386543823805,0.18282767194851973,1000 diff --git a/results/irm/irm_atte_coverage_metadata.csv b/results/irm/irm_atte_coverage_metadata.csv index b9f90c7..18af13a 100644 --- a/results/irm/irm_atte_coverage_metadata.csv +++ b/results/irm/irm_atte_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.9.0,irm_atte_coverage.py,2024-09-09 08:23:18,3582.4482469558716,3.12.5 +0.10.dev0,irm_atte_coverage.py,2025-01-08 13:14:10,3538.650551557541,3.12.8 diff --git a/results/irm/irm_atte_sensitivity.csv b/results/irm/irm_atte_sensitivity.csv index cd6a85e..7483e17 100644 --- a/results/irm/irm_atte_sensitivity.csv +++ b/results/irm/irm_atte_sensitivity.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),repetition LGBM,LGBM,0.9,0.702,0.348892741328716,0.13535473129404849,0.95,1.0,0.10509293446782587,0.024288450266824614,0.0649958270073569,0.25876253126104387,500 LGBM,LGBM,0.95,0.826,0.41573134306126747,0.13535473129404849,0.982,1.0,0.10509293446782587,0.012452066983782133,0.0649958270073569,0.25876253126104387,500 -LGBM,Logistic Regr.,0.9,0.714,0.34666910502599596,0.13078975736827733,0.964,1.0,0.09817018528214369,0.02224699823797256,0.0654544233024662,0.2589976808508975,500 -LGBM,Logistic Regr.,0.95,0.834,0.41308171698108886,0.13078975736827733,0.984,1.0,0.09817018528214369,0.010949342084431749,0.0654544233024662,0.2589976808508975,500 +LGBM,Logistic Regr.,0.9,0.714,0.34666910502599596,0.13078975736827733,0.964,1.0,0.09817018528214369,0.022246998237972562,0.06545442330246619,0.2589976808508975,500 +LGBM,Logistic Regr.,0.95,0.834,0.41308171698108886,0.13078975736827733,0.984,1.0,0.09817018528214369,0.010949342084431752,0.06545442330246619,0.2589976808508975,500 Linear Reg.,LGBM,0.9,0.754,0.3496967006881292,0.12455057551341779,0.962,1.0,0.09867724125956986,0.02021759355041513,0.06504946816195573,0.24393419014571052,500 Linear Reg.,LGBM,0.95,0.858,0.416689319724762,0.12455057551341779,0.986,1.0,0.09867724125956986,0.009856683129418061,0.06504946816195573,0.24393419014571052,500 -Linear Reg.,Logistic Regr.,0.9,0.948,0.3502540540945954,0.07444772768321124,0.996,1.0,0.0584014583662732,0.004181143741279687,0.09544484272838329,0.17545346180009289,500 -Linear Reg.,Logistic Regr.,0.95,0.976,0.41735344727108903,0.07444772768321124,0.998,1.0,0.0584014583662732,0.001573924919578164,0.09544484272838329,0.17545346180009289,500 +Linear Reg.,Logistic Regr.,0.9,0.948,0.3502540540945954,0.07444772768321124,0.996,1.0,0.05840145836627319,0.0041811437412796835,0.09544484272838329,0.17545346180009289,500 +Linear Reg.,Logistic Regr.,0.95,0.976,0.41735344727108903,0.07444772768321124,0.998,1.0,0.05840145836627319,0.001573924919578166,0.09544484272838329,0.17545346180009289,500 diff --git a/results/irm/irm_atte_sensitivity_metadata.csv b/results/irm/irm_atte_sensitivity_metadata.csv index 09e20a9..4353523 100644 --- a/results/irm/irm_atte_sensitivity_metadata.csv +++ b/results/irm/irm_atte_sensitivity_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.9.0,irm_atte_sensitivity.py,2024-09-09 10:03:27,9588.962257862091,3.12.5 +0.10.dev0,irm_atte_sensitivity.py,2025-01-08 15:14:53,10782.067339658737,3.12.8 diff --git a/results/irm/irm_cate_coverage.csv b/results/irm/irm_cate_coverage.csv index 3d56bac..42b5454 100644 --- a/results/irm/irm_cate_coverage.csv +++ b/results/irm/irm_cate_coverage.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition LGBM,LGBM,0.9,0.9356475000000001,0.6670555194228769,0.1482947174222056,1.0,1.689294724011131,1000 LGBM,LGBM,0.95,0.9723379999999999,0.7948456764390678,0.1482947174222056,1.0,1.6897571332817831,1000 -LGBM,Logistic Regression,0.9,0.8889914999999999,0.23544529209287568,0.058506993145254954,0.996,0.595737920698468,1000 -LGBM,Logistic Regression,0.95,0.942037,0.2805503695102737,0.058506993145254954,0.996,0.5990789624833717,1000 +LGBM,Logistic Regression,0.9,0.8889914999999999,0.2354452920928757,0.05850699314525499,0.996,0.5957379206984681,1000 +LGBM,Logistic Regression,0.95,0.942037,0.28055036951027373,0.05850699314525499,0.996,0.5990789624833718,1000 Lasso,LGBM,0.9,0.896289,0.6428742805642967,0.15829333167540963,1.0,1.6320898187101593,1000 Lasso,LGBM,0.95,0.9491539999999999,0.7660319531461224,0.15829333167540963,1.0,1.6337799284392311,1000 -Lasso,Logistic Regression,0.9,0.8892920000000001,0.24738399667726235,0.061668240816651954,0.998,0.6298396366786109,1000 -Lasso,Logistic Regression,0.95,0.9413365,0.2947762134541078,0.061668240816651954,0.997,0.6288417962868702,1000 +Lasso,Logistic Regression,0.9,0.8892920000000001,0.2473839966772624,0.061668240816652016,0.998,0.629839636678611,1000 +Lasso,Logistic Regression,0.95,0.9413365,0.29477621345410787,0.061668240816652016,0.997,0.6288417962868702,1000 diff --git a/results/irm/irm_cate_coverage_metadata.csv b/results/irm/irm_cate_coverage_metadata.csv index a9b76cb..24d54ab 100644 --- a/results/irm/irm_cate_coverage_metadata.csv +++ b/results/irm/irm_cate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.9.0,irm_cate_coverage.py,2024-09-09 10:07:30,9833.500946998596,3.12.5 +0.10.dev0,irm_cate_coverage.py,2025-01-08 14:19:05,7429.032528162003,3.12.8 diff --git a/results/irm/irm_gate_coverage.csv b/results/irm/irm_gate_coverage.csv index f9fe660..f5dde96 100644 --- a/results/irm/irm_gate_coverage.csv +++ b/results/irm/irm_gate_coverage.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition LGBM,LGBM,0.9,0.941,2.1611526373873855,0.48511146422993706,1.0,5.063646757352436,1000 LGBM,LGBM,0.95,0.9766666666666667,2.5751722007164304,0.48511146422993706,1.0,5.085406768554818,1000 -LGBM,Logistic Regression,0.9,0.916,0.39042806875814784,0.08854865701618221,0.997,0.9186129412168422,1000 -LGBM,Logistic Regression,0.95,0.9606666666666667,0.4652237383199531,0.08854865701618221,0.998,0.9197763724365766,1000 +LGBM,Logistic Regression,0.9,0.916,0.3904280687581478,0.08854865701618204,0.997,0.9186129412168421,1000 +LGBM,Logistic Regression,0.95,0.9606666666666667,0.4652237383199531,0.08854865701618204,0.998,0.9197763724365766,1000 Lasso,LGBM,0.9,0.9043333333333333,2.047596498760145,0.49693265710872336,1.0,4.807194790243196,1000 Lasso,LGBM,0.95,0.959,2.43986171576749,0.49693265710872336,1.0,4.819825928994352,1000 -Lasso,Logistic Regression,0.9,0.9173333333333333,0.4009671672064071,0.08997238623406144,0.999,0.9416356317140333,1000 -Lasso,Logistic Regression,0.95,0.9603333333333334,0.47778184868895535,0.08997238623406144,0.999,0.9403831822862943,1000 +Lasso,Logistic Regression,0.9,0.9173333333333333,0.400967167206407,0.08997238623406502,0.999,0.9416356317140329,1000 +Lasso,Logistic Regression,0.95,0.9603333333333334,0.4777818486889552,0.08997238623406502,0.999,0.9403831822862944,1000 diff --git a/results/irm/irm_gate_coverage_metadata.csv b/results/irm/irm_gate_coverage_metadata.csv index 3e1d4f2..b7034be 100644 --- a/results/irm/irm_gate_coverage_metadata.csv +++ b/results/irm/irm_gate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.9.0,irm_gate_coverage.py,2024-09-09 08:26:21,3764.0360503196716,3.12.5 +0.10.dev0,irm_gate_coverage.py,2025-01-08 13:08:04,3153.3980271816254,3.12.8 diff --git a/results/irm/lpq_coverage_lpq0.csv b/results/irm/lpq_coverage_lpq0.csv index 8df271f..566dd1f 100644 --- a/results/irm/lpq_coverage_lpq0.csv +++ b/results/irm/lpq_coverage_lpq0.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.9,0.94,1.1717023704378204,0.23183406397860207,100 -LGBM,LGBM,0.95,0.9733333333333333,1.396169488293374,0.23183406397860207,100 -LGBM,Logistic Regression,0.9,0.95,1.124933562445929,0.2275644497861777,100 -LGBM,Logistic Regression,0.95,0.9766666666666667,1.3404410163114224,0.2275644497861777,100 -Logistic Regression,LGBM,0.9,0.9477777777777777,1.1516756381210262,0.22097742768241296,100 -Logistic Regression,LGBM,0.95,0.9811111111111112,1.3723061648791872,0.22097742768241296,100 -Logistic Regression,Logistic Regression,0.9,0.9533333333333333,1.1085912475488997,0.20668700656290437,100 -Logistic Regression,Logistic Regression,0.95,0.9788888888888888,1.3209679470380462,0.20668700656290437,100 +LGBM,LGBM,0.9,0.94,1.1717023704378204,0.23177376824257415,100 +LGBM,LGBM,0.95,0.9722222222222223,1.396169488293374,0.23177376824257415,100 +LGBM,Logistic Regression,0.9,0.95,1.1248327860637255,0.2271354130168218,100 +LGBM,Logistic Regression,0.95,0.9755555555555556,1.3403209338454969,0.2271354130168218,100 +Logistic Regression,LGBM,0.9,0.95,1.1516756381210271,0.22092321662449366,100 +Logistic Regression,LGBM,0.95,0.9811111111111112,1.372306164879188,0.22092321662449366,100 +Logistic Regression,Logistic Regression,0.9,0.9533333333333333,1.1095156391492578,0.20615547550299898,100 +Logistic Regression,Logistic Regression,0.95,0.9788888888888888,1.3220694275677582,0.20615547550299898,100 diff --git a/results/irm/lpq_coverage_lpq1.csv b/results/irm/lpq_coverage_lpq1.csv index 6bfc9b2..ba57b07 100644 --- a/results/irm/lpq_coverage_lpq1.csv +++ b/results/irm/lpq_coverage_lpq1.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.9,0.9577777777777777,1.6624989751251327,0.32794811978077126,100 -LGBM,LGBM,0.95,0.99,1.9809897137285786,0.32794811978077126,100 -LGBM,Logistic Regression,0.9,0.94,1.589606393126724,0.3222751957636039,100 -LGBM,Logistic Regression,0.95,0.97,1.8941328450588726,0.3222751957636039,100 -Logistic Regression,LGBM,0.9,0.96,1.6078485820230732,0.3009359326740972,100 -Logistic Regression,LGBM,0.95,0.9822222222222223,1.9158697538331122,0.3009359326740972,100 -Logistic Regression,Logistic Regression,0.9,0.9466666666666668,1.5564190145145989,0.3055460305222601,100 -Logistic Regression,Logistic Regression,0.95,0.9844444444444445,1.8545876443460196,0.3055460305222601,100 +LGBM,LGBM,0.9,0.9577777777777777,1.6624989751251327,0.32781267630898475,100 +LGBM,LGBM,0.95,0.99,1.9809897137285786,0.32781267630898475,100 +LGBM,Logistic Regression,0.9,0.9411111111111111,1.5914676830888603,0.32199246221295674,100 +LGBM,Logistic Regression,0.95,0.97,1.8963507088437095,0.32199246221295674,100 +Logistic Regression,LGBM,0.9,0.96,1.60784858202307,0.30086810092875493,100 +Logistic Regression,LGBM,0.95,0.9822222222222223,1.915869753833108,0.30086810092875493,100 +Logistic Regression,Logistic Regression,0.9,0.9477777777777777,1.5561898364166578,0.30507352946060423,100 +Logistic Regression,Logistic Regression,0.95,0.9844444444444445,1.854314561798947,0.30507352946060423,100 diff --git a/results/irm/lpq_coverage_lqte.csv b/results/irm/lpq_coverage_lqte.csv index 31322fc..4edaa4e 100644 --- a/results/irm/lpq_coverage_lqte.csv +++ b/results/irm/lpq_coverage_lqte.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.9233333333333333,1.636872420792144,0.38509449105953003,0.94,2.2547716211801454,100 -LGBM,LGBM,0.95,0.9633333333333333,1.950453790824845,0.38509449105953003,0.97,2.5400452907969764,100 -LGBM,Logistic Regression,0.9,0.91,1.5657795728691977,0.37487491739204254,0.9,2.154808632448696,100 -LGBM,Logistic Regression,0.95,0.9488888888888889,1.8657414375769725,0.37487491739204254,0.95,2.424204209936085,100 -Logistic Regression,LGBM,0.9,0.9188888888888889,1.6190226009242594,0.3724923151955094,0.92,2.208051172256097,100 -Logistic Regression,LGBM,0.95,0.9644444444444444,1.9291844185850662,0.3724923151955094,0.98,2.487384422889674,100 -Logistic Regression,Logistic Regression,0.9,0.9033333333333333,1.563967614483271,0.37277891050263284,0.9,2.13603227354138,100 -Logistic Regression,Logistic Regression,0.95,0.9488888888888889,1.8635823559907991,0.37277891050263284,0.95,2.4084686120659855,100 +LGBM,LGBM,0.9,0.9233333333333333,1.636872420792144,0.3849163075483558,0.94,2.2547716211801454,100 +LGBM,LGBM,0.95,0.9633333333333333,1.950453790824845,0.3849163075483558,0.97,2.5400452907969764,100 +LGBM,Logistic Regression,0.9,0.9077777777777777,1.5675486277219108,0.3750309062078869,0.91,2.157430608143212,100 +LGBM,Logistic Regression,0.95,0.95,1.8678493964501393,0.3750309062078869,0.95,2.4273382603132627,100 +Logistic Regression,LGBM,0.9,0.9188888888888889,1.6190226009242576,0.3723079594843724,0.92,2.208051172256097,100 +Logistic Regression,LGBM,0.95,0.9644444444444444,1.9291844185850642,0.3723079594843724,0.98,2.4873844228896727,100 +Logistic Regression,Logistic Regression,0.9,0.9077777777777777,1.564249778556274,0.37200263232334746,0.9,2.1368366984439215,100 +Logistic Regression,Logistic Regression,0.95,0.9477777777777777,1.8639185752213465,0.37200263232334746,0.95,2.4083334749081065,100 diff --git a/results/irm/lpq_coverage_metadata.csv b/results/irm/lpq_coverage_metadata.csv index 8571998..63994d8 100644 --- a/results/irm/lpq_coverage_metadata.csv +++ b/results/irm/lpq_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.9.0,lpq_coverage.py,2024-09-09 12:37:02,18799.444734811783,3.12.5 +0.10.dev0,lpq_coverage.py,2025-01-08 17:24:33,18545.282371282578,3.12.8 diff --git a/results/irm/pq_coverage_metadata.csv b/results/irm/pq_coverage_metadata.csv index ca61794..17b95a8 100644 --- a/results/irm/pq_coverage_metadata.csv +++ b/results/irm/pq_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.9.0,pq_coverage.py,2024-09-09 12:24:40,18061.192512512207,3.12.5 +0.10.dev0,pq_coverage.py,2025-01-08 17:13:20,17871.711226463318,3.12.8 diff --git a/results/irm/pq_coverage_pq0.csv b/results/irm/pq_coverage_pq0.csv index 87e1c2c..c934b1a 100644 --- a/results/irm/pq_coverage_pq0.csv +++ b/results/irm/pq_coverage_pq0.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.9,0.8823076923076922,0.5804011224570385,0.1491025005152581,100 -LGBM,LGBM,0.95,0.9346153846153846,0.6915905938151775,0.1491025005152581,100 -LGBM,Logistic Regression,0.9,0.8115384615384617,0.3861407256651673,0.11416614656762226,100 -LGBM,Logistic Regression,0.95,0.8861538461538462,0.46011505392766316,0.11416614656762226,100 -Logistic Regression,LGBM,0.9,0.8853846153846153,0.5879335180942484,0.15100690944933073,100 -Logistic Regression,LGBM,0.95,0.9415384615384617,0.7005659968080868,0.15100690944933073,100 -Logistic Regression,Logistic Regression,0.9,0.8223076923076923,0.38942866423036987,0.1123937956401062,100 -Logistic Regression,Logistic Regression,0.95,0.9015384615384616,0.46403287437416774,0.1123937956401062,100 +LGBM,LGBM,0.9,0.8838461538461538,0.5804011224570385,0.1489863706604989,100 +LGBM,LGBM,0.95,0.9353846153846154,0.6915905938151775,0.1489863706604989,100 +LGBM,Logistic Regression,0.9,0.8123076923076923,0.3860909559387072,0.11389794833419535,100 +LGBM,Logistic Regression,0.95,0.8876923076923078,0.46005574964077484,0.11389794833419535,100 +Logistic Regression,LGBM,0.9,0.8861538461538462,0.5879335180942487,0.1508479802166077,100 +Logistic Regression,LGBM,0.95,0.9423076923076923,0.7005659968080872,0.1508479802166077,100 +Logistic Regression,Logistic Regression,0.9,0.8246153846153846,0.3895072188527442,0.1120574898221174,100 +Logistic Regression,Logistic Regression,0.95,0.9046153846153847,0.4641264779800753,0.1120574898221174,100 diff --git a/results/irm/pq_coverage_pq1.csv b/results/irm/pq_coverage_pq1.csv index 5e8e4fc..a56309b 100644 --- a/results/irm/pq_coverage_pq1.csv +++ b/results/irm/pq_coverage_pq1.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.9,0.9130769230769231,0.2504138111093655,0.05865662571787472,100 -LGBM,LGBM,0.95,0.9623076923076923,0.298386460025268,0.05865662571787472,100 -LGBM,Logistic Regression,0.9,0.91,0.22948465548607,0.05375633654188718,100 -LGBM,Logistic Regression,0.95,0.9615384615384616,0.2734478329180528,0.05375633654188718,100 -Logistic Regression,LGBM,0.9,0.9192307692307692,0.25410039964172587,0.059363532936649595,100 -Logistic Regression,LGBM,0.95,0.9607692307692308,0.3027793012063014,0.059363532936649595,100 -Logistic Regression,Logistic Regression,0.9,0.8984615384615385,0.23093353063439134,0.057191896173723104,100 -Logistic Regression,Logistic Regression,0.95,0.9553846153846153,0.27517427414192513,0.057191896173723104,100 +LGBM,LGBM,0.9,0.9138461538461539,0.2504138111093655,0.058644803131633845,100 +LGBM,LGBM,0.95,0.9607692307692308,0.298386460025268,0.058644803131633845,100 +LGBM,Logistic Regression,0.9,0.9130769230769231,0.2294853436068611,0.05383096701300778,100 +LGBM,Logistic Regression,0.95,0.9623076923076923,0.2734486528645486,0.05383096701300778,100 +Logistic Regression,LGBM,0.9,0.916923076923077,0.25410039964172587,0.059305556876141904,100 +Logistic Regression,LGBM,0.95,0.9615384615384616,0.3027793012063014,0.059305556876141904,100 +Logistic Regression,Logistic Regression,0.9,0.8976923076923078,0.23093595343208725,0.057241427868032047,100 +Logistic Regression,Logistic Regression,0.95,0.9546153846153846,0.27517716108344265,0.057241427868032047,100 diff --git a/results/irm/pq_coverage_qte.csv b/results/irm/pq_coverage_qte.csv index 4fea39b..a0fec02 100644 --- a/results/irm/pq_coverage_qte.csv +++ b/results/irm/pq_coverage_qte.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.8815384615384616,0.6176207568284253,0.15504772624658503,0.85,0.9336715877443045,100 -LGBM,LGBM,0.95,0.9407692307692308,0.7359405236146267,0.15504772624658503,0.91,1.0306538899948843,100 -LGBM,Logistic Regression,0.9,0.83,0.4257824194261344,0.12245088179297632,0.78,0.6460154145368233,100 -LGBM,Logistic Regression,0.95,0.9084615384615385,0.5073510454983309,0.12245088179297632,0.84,0.714250944722706,100 -Logistic Regression,LGBM,0.9,0.8961538461538461,0.6286198118927269,0.15900405481444213,0.87,0.9308233463178206,100 -Logistic Regression,LGBM,0.95,0.9484615384615386,0.7490467060960178,0.15900405481444213,0.92,1.0293001363085716,100 -Logistic Regression,Logistic Regression,0.9,0.8430769230769231,0.4344065634613077,0.12399216449911891,0.81,0.647334882587494,100 -Logistic Regression,Logistic Regression,0.95,0.9115384615384616,0.5176273469451368,0.12399216449911891,0.86,0.7197477669291079,100 +LGBM,LGBM,0.9,0.8815384615384616,0.6176207568284253,0.15515498246785267,0.85,0.9336715877443045,100 +LGBM,LGBM,0.95,0.94,0.7359405236146267,0.15515498246785267,0.91,1.0306538899948843,100 +LGBM,Logistic Regression,0.9,0.826923076923077,0.4257337159670842,0.122640423715187,0.78,0.6459408206266708,100 +LGBM,Logistic Regression,0.95,0.9092307692307692,0.5072930117474264,0.122640423715187,0.84,0.7141895123082023,100 +Logistic Regression,LGBM,0.9,0.8969230769230769,0.6286198118927271,0.15916890224887578,0.87,0.9308233463178209,100 +Logistic Regression,LGBM,0.95,0.946923076923077,0.7490467060960181,0.15916890224887578,0.92,1.029300136308572,100 +Logistic Regression,Logistic Regression,0.9,0.8407692307692308,0.43448201352405663,0.12418740801387926,0.81,0.6474698211016049,100 +Logistic Regression,Logistic Regression,0.95,0.9123076923076923,0.5177172512400822,0.12418740801387926,0.85,0.719895714650183,100 diff --git a/results/irm/ssm_mar_ate_coverage.csv b/results/irm/ssm_mar_ate_coverage.csv index 5a9014d..59a4f8d 100644 --- a/results/irm/ssm_mar_ate_coverage.csv +++ b/results/irm/ssm_mar_ate_coverage.csv @@ -1,17 +1,17 @@ Learner g,Learner m,Learner pi,score,level,Coverage,CI Length,Bias,repetition LGBM,LGBM,LGBM,missing-at-random,0.9,0.934,5.894613599209654,1.524603484598883,1000 LGBM,LGBM,LGBM,missing-at-random,0.95,0.981,7.023865326329001,1.524603484598883,1000 -LGBM,LGBM,Logistic,missing-at-random,0.9,0.927,2.5815888675562886,0.6151192173080173,1000 -LGBM,LGBM,Logistic,missing-at-random,0.95,0.973,3.0761528687981845,0.6151192173080173,1000 -LGBM,Logistic,LGBM,missing-at-random,0.9,0.945,2.5756414334122404,0.6542029876050892,1000 -LGBM,Logistic,LGBM,missing-at-random,0.95,0.985,3.06906606391065,0.6542029876050892,1000 -LGBM,Logistic,Logistic,missing-at-random,0.9,0.914,0.5399737109966924,0.12672360617482684,1000 -LGBM,Logistic,Logistic,missing-at-random,0.95,0.958,0.6434183618596117,0.12672360617482684,1000 +LGBM,LGBM,Logistic,missing-at-random,0.9,0.927,2.581588867556288,0.6151192173080174,1000 +LGBM,LGBM,Logistic,missing-at-random,0.95,0.973,3.0761528687981836,0.6151192173080174,1000 +LGBM,Logistic,LGBM,missing-at-random,0.9,0.945,2.5756414334122413,0.6542029876050892,1000 +LGBM,Logistic,LGBM,missing-at-random,0.95,0.985,3.0690660639106513,0.6542029876050892,1000 +LGBM,Logistic,Logistic,missing-at-random,0.9,0.914,0.5399737109966929,0.12672360617482706,1000 +LGBM,Logistic,Logistic,missing-at-random,0.95,0.958,0.6434183618596122,0.12672360617482706,1000 Lasso,LGBM,LGBM,missing-at-random,0.9,0.939,5.030966633759897,1.2700343898139361,1000 Lasso,LGBM,LGBM,missing-at-random,0.95,0.981,5.994766493519136,1.2700343898139361,1000 -Lasso,LGBM,Logistic,missing-at-random,0.9,0.887,2.341425282643257,0.6221402191258445,1000 -Lasso,LGBM,Logistic,missing-at-random,0.95,0.955,2.7899803066231694,0.6221402191258445,1000 -Lasso,Logistic,LGBM,missing-at-random,0.9,0.919,2.2995632695758172,0.6130089902400221,1000 -Lasso,Logistic,LGBM,missing-at-random,0.95,0.97,2.7400986414171333,0.6130089902400221,1000 -Lasso,Logistic,Logistic,missing-at-random,0.9,0.897,0.5116626905293394,0.12267262949094226,1000 -Lasso,Logistic,Logistic,missing-at-random,0.95,0.961,0.609683700262744,0.12267262949094226,1000 +Lasso,LGBM,Logistic,missing-at-random,0.9,0.887,2.3414252826432578,0.6221402191258446,1000 +Lasso,LGBM,Logistic,missing-at-random,0.95,0.955,2.78998030662317,0.6221402191258446,1000 +Lasso,Logistic,LGBM,missing-at-random,0.9,0.919,2.2995632695758177,0.6130089902400226,1000 +Lasso,Logistic,LGBM,missing-at-random,0.95,0.97,2.7400986414171338,0.6130089902400226,1000 +Lasso,Logistic,Logistic,missing-at-random,0.9,0.897,0.5116626905293399,0.12267262949094253,1000 +Lasso,Logistic,Logistic,missing-at-random,0.95,0.961,0.6096837002627445,0.12267262949094253,1000 diff --git a/results/irm/ssm_mar_ate_coverage_metadata.csv b/results/irm/ssm_mar_ate_coverage_metadata.csv index 8ff1d7a..7a7fba8 100644 --- a/results/irm/ssm_mar_ate_coverage_metadata.csv +++ b/results/irm/ssm_mar_ate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.9.0,ssm_mar_ate_coverage.py,2024-09-09 09:44:28,8451.34564447403,3.12.5 +0.10.dev0,ssm_mar_ate_coverage.py,2025-01-08 14:14:00,7131.662969589233,3.12.8 diff --git a/results/irm/ssm_nonignorable_ate_coverage.csv b/results/irm/ssm_nonignorable_ate_coverage.csv index 6d5265a..f85fc53 100644 --- a/results/irm/ssm_nonignorable_ate_coverage.csv +++ b/results/irm/ssm_nonignorable_ate_coverage.csv @@ -1,17 +1,17 @@ Learner g,Learner m,Learner pi,score,level,Coverage,CI Length,Bias,repetition LGBM,LGBM,LGBM,nonignorable,0.9,0.919,13.007296321798503,3.6668819438868456,1000 LGBM,LGBM,LGBM,nonignorable,0.95,0.966,15.499149534791721,3.6668819438868456,1000 -LGBM,LGBM,Logistic,nonignorable,0.9,0.918,4.832148171492925,1.3269099332105847,1000 -LGBM,LGBM,Logistic,nonignorable,0.95,0.974,5.757859683624376,1.3269099332105847,1000 -LGBM,Logistic,LGBM,nonignorable,0.9,0.897,4.592630929679537,1.2319933070953284,1000 -LGBM,Logistic,LGBM,nonignorable,0.95,0.959,5.472457286755379,1.2319933070953284,1000 -LGBM,Logistic,Logistic,nonignorable,0.9,0.867,2.530181172735919,0.7278088971445482,1000 -LGBM,Logistic,Logistic,nonignorable,0.95,0.94,3.0148968222264956,0.7278088971445482,1000 +LGBM,LGBM,Logistic,nonignorable,0.9,0.918,4.832148171492935,1.326909933210586,1000 +LGBM,LGBM,Logistic,nonignorable,0.95,0.974,5.757859683624388,1.326909933210586,1000 +LGBM,Logistic,LGBM,nonignorable,0.9,0.897,4.592630929679517,1.2319933070953264,1000 +LGBM,Logistic,LGBM,nonignorable,0.95,0.959,5.472457286755355,1.2319933070953264,1000 +LGBM,Logistic,Logistic,nonignorable,0.9,0.867,2.5301811727355052,0.7278088971443857,1000 +LGBM,Logistic,Logistic,nonignorable,0.95,0.94,3.014896822226002,0.7278088971443857,1000 Lasso,LGBM,LGBM,nonignorable,0.9,0.931,10.203902842326483,2.892343371671614,1000 Lasso,LGBM,LGBM,nonignorable,0.95,0.975,12.158700169432056,2.892343371671614,1000 -Lasso,LGBM,Logistic,nonignorable,0.9,0.917,7.0625497859220685,2.0505032981783837,1000 -Lasso,LGBM,Logistic,nonignorable,0.95,0.974,8.415547129919004,2.0505032981783837,1000 -Lasso,Logistic,LGBM,nonignorable,0.9,0.894,4.170596833405643,1.1867597841647153,1000 -Lasso,Logistic,LGBM,nonignorable,0.95,0.95,4.969572643774837,1.1867597841647153,1000 -Lasso,Logistic,Logistic,nonignorable,0.9,0.87,1.9933166328366938,0.5662899386624944,1000 -Lasso,Logistic,Logistic,nonignorable,0.95,0.928,2.3751832662371193,0.5662899386624944,1000 +Lasso,LGBM,Logistic,nonignorable,0.9,0.917,7.062549785922076,2.0505032981783877,1000 +Lasso,LGBM,Logistic,nonignorable,0.95,0.974,8.415547129919013,2.0505032981783877,1000 +Lasso,Logistic,LGBM,nonignorable,0.9,0.894,4.170596833405628,1.1867597841647148,1000 +Lasso,Logistic,LGBM,nonignorable,0.95,0.95,4.969572643774819,1.1867597841647148,1000 +Lasso,Logistic,Logistic,nonignorable,0.9,0.87,1.9933166328368466,0.5662899386625126,1000 +Lasso,Logistic,Logistic,nonignorable,0.95,0.928,2.375183266237302,0.5662899386625126,1000 diff --git a/results/irm/ssm_nonignorable_ate_coverage_metadata.csv b/results/irm/ssm_nonignorable_ate_coverage_metadata.csv index c0ae92d..56ee090 100644 --- a/results/irm/ssm_nonignorable_ate_coverage_metadata.csv +++ b/results/irm/ssm_nonignorable_ate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.9.0,ssm_nonignorable_ate_coverage.py,2024-09-09 08:50:13,5195.712763547897,3.12.5 +0.10.dev0,ssm_nonignorable_ate_coverage.py,2025-01-08 14:24:21,7745.532069444656,3.12.8 diff --git a/results/plm/pliv_late_coverage.csv b/results/plm/pliv_late_coverage.csv index 0482f22..eec541b 100644 --- a/results/plm/pliv_late_coverage.csv +++ b/results/plm/pliv_late_coverage.csv @@ -3,31 +3,31 @@ Lasso,Lasso,Lasso,IV-type,0.9,0.806,0.23181691237656843,0.071516051616331,1000 Lasso,Lasso,Lasso,IV-type,0.95,0.873,0.2762268884116072,0.071516051616331,1000 Lasso,Lasso,Lasso,partialling out,0.9,0.899,0.2994589442321557,0.07151348023026591,1000 Lasso,Lasso,Lasso,partialling out,0.95,0.947,0.35682734069852273,0.07151348023026591,1000 -Lasso,Lasso,Random Forest,IV-type,0.9,0.806,0.23192478502357472,0.07187026721767026,1000 -Lasso,Lasso,Random Forest,IV-type,0.95,0.865,0.2763554265985832,0.07187026721767026,1000 -Lasso,Lasso,Random Forest,partialling out,0.9,0.896,0.3068383735438992,0.0741569646182831,1000 -Lasso,Lasso,Random Forest,partialling out,0.95,0.947,0.3656204730724243,0.0741569646182831,1000 -Lasso,Random Forest,Lasso,IV-type,0.9,0.83,0.2644292585766018,0.07753973367612393,1000 -Lasso,Random Forest,Lasso,IV-type,0.95,0.901,0.3150868957436169,0.07753973367612393,1000 -Lasso,Random Forest,Lasso,partialling out,0.9,0.904,0.3173435440931062,0.07691371311292067,1000 -Lasso,Random Forest,Lasso,partialling out,0.95,0.956,0.3781381558561849,0.07691371311292067,1000 -Lasso,Random Forest,Random Forest,IV-type,0.9,0.838,0.26484483955141147,0.07664037514487268,1000 -Lasso,Random Forest,Random Forest,IV-type,0.95,0.909,0.31558209101810264,0.07664037514487268,1000 -Lasso,Random Forest,Random Forest,partialling out,0.9,0.876,0.3009573997300998,0.08173024901795378,1000 -Lasso,Random Forest,Random Forest,partialling out,0.95,0.943,0.3586128605528636,0.08173024901795378,1000 -Random Forest,Lasso,Lasso,IV-type,0.9,0.792,0.24229814878232575,0.07640585085746515,1000 -Random Forest,Lasso,Lasso,IV-type,0.95,0.867,0.28871605190441474,0.07640585085746515,1000 -Random Forest,Lasso,Lasso,partialling out,0.9,0.908,0.3309481527556741,0.08063082596936634,1000 -Random Forest,Lasso,Lasso,partialling out,0.95,0.952,0.39434904694429584,0.08063082596936634,1000 -Random Forest,Lasso,Random Forest,IV-type,0.9,0.802,0.24169974855188303,0.07499899652306129,1000 -Random Forest,Lasso,Random Forest,IV-type,0.95,0.875,0.28800301405059553,0.07499899652306129,1000 -Random Forest,Lasso,Random Forest,partialling out,0.9,0.912,0.31892872567199654,0.07590450344220336,1000 -Random Forest,Lasso,Random Forest,partialling out,0.95,0.958,0.3800270162098806,0.07590450344220336,1000 -Random Forest,Random Forest,Lasso,IV-type,0.9,0.826,0.27840682771822006,0.07984522594527775,1000 -Random Forest,Random Forest,Lasso,IV-type,0.95,0.888,0.33174219665313587,0.07984522594527775,1000 -Random Forest,Random Forest,Lasso,partialling out,0.9,0.79,0.35115828845926267,0.10762971655091783,1000 -Random Forest,Random Forest,Lasso,partialling out,0.95,0.865,0.4184309089730254,0.10762971655091783,1000 -Random Forest,Random Forest,Random Forest,IV-type,0.9,0.822,0.27425117516011144,0.07888472952601593,1000 -Random Forest,Random Forest,Random Forest,IV-type,0.95,0.893,0.32679043121170237,0.07888472952601593,1000 -Random Forest,Random Forest,Random Forest,partialling out,0.9,0.874,0.30385637054427705,0.07807502303728009,1000 -Random Forest,Random Forest,Random Forest,partialling out,0.95,0.93,0.3620671973369523,0.07807502303728009,1000 +Lasso,Lasso,Random Forest,IV-type,0.9,0.808,0.23192003415034532,0.07189717055113612,1000 +Lasso,Lasso,Random Forest,IV-type,0.95,0.866,0.27634976558396634,0.07189717055113612,1000 +Lasso,Lasso,Random Forest,partialling out,0.9,0.899,0.306946616292115,0.0739728160311134,1000 +Lasso,Lasso,Random Forest,partialling out,0.95,0.947,0.365749452262192,0.0739728160311134,1000 +Lasso,Random Forest,Lasso,IV-type,0.9,0.83,0.2641727277684909,0.07763295491460846,1000 +Lasso,Random Forest,Lasso,IV-type,0.95,0.9,0.31478122043209744,0.07763295491460846,1000 +Lasso,Random Forest,Lasso,partialling out,0.9,0.906,0.31724103561902756,0.07721936392175144,1000 +Lasso,Random Forest,Lasso,partialling out,0.95,0.952,0.3780160094754904,0.07721936392175144,1000 +Lasso,Random Forest,Random Forest,IV-type,0.9,0.834,0.2647124176667093,0.07680126322014207,1000 +Lasso,Random Forest,Random Forest,IV-type,0.95,0.914,0.31542430061017296,0.07680126322014207,1000 +Lasso,Random Forest,Random Forest,partialling out,0.9,0.874,0.3008847110321322,0.08096376656362451,1000 +Lasso,Random Forest,Random Forest,partialling out,0.95,0.948,0.358526246627001,0.08096376656362451,1000 +Random Forest,Lasso,Lasso,IV-type,0.9,0.797,0.24216323960269195,0.07658572792546856,1000 +Random Forest,Lasso,Lasso,IV-type,0.95,0.864,0.2885552977017711,0.07658572792546856,1000 +Random Forest,Lasso,Lasso,partialling out,0.9,0.905,0.33082304355897396,0.08047007130194417,1000 +Random Forest,Lasso,Lasso,partialling out,0.95,0.954,0.3941999701415645,0.08047007130194417,1000 +Random Forest,Lasso,Random Forest,IV-type,0.9,0.802,0.24165696675117043,0.07510539234046823,1000 +Random Forest,Lasso,Random Forest,IV-type,0.95,0.878,0.28795203639081074,0.07510539234046823,1000 +Random Forest,Lasso,Random Forest,partialling out,0.9,0.905,0.31918216721413517,0.07671818026815334,1000 +Random Forest,Lasso,Random Forest,partialling out,0.95,0.949,0.3803290104339495,0.07671818026815334,1000 +Random Forest,Random Forest,Lasso,IV-type,0.9,0.828,0.27855600069164016,0.0798644714171535,1000 +Random Forest,Random Forest,Lasso,IV-type,0.95,0.894,0.33191994721438917,0.0798644714171535,1000 +Random Forest,Random Forest,Lasso,partialling out,0.9,0.792,0.35144455288760573,0.1068047543860758,1000 +Random Forest,Random Forest,Lasso,partialling out,0.95,0.862,0.418772014078315,0.1068047543860758,1000 +Random Forest,Random Forest,Random Forest,IV-type,0.9,0.819,0.27446473521048453,0.07996992581732235,1000 +Random Forest,Random Forest,Random Forest,IV-type,0.95,0.889,0.3270449037072543,0.07996992581732235,1000 +Random Forest,Random Forest,Random Forest,partialling out,0.9,0.866,0.30400744761250564,0.07843878847708105,1000 +Random Forest,Random Forest,Random Forest,partialling out,0.95,0.926,0.3622472167671108,0.07843878847708105,1000 diff --git a/results/plm/pliv_late_coverage_metadata.csv b/results/plm/pliv_late_coverage_metadata.csv index d631247..37383e7 100644 --- a/results/plm/pliv_late_coverage_metadata.csv +++ b/results/plm/pliv_late_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.9.0,pliv_late_coverage.py,2024-09-09 12:44:11,19234.77849960327,3.12.5 +0.10.dev0,pliv_late_coverage.py,2025-01-08 17:34:57,19185.31809258461,3.12.8 diff --git a/results/plm/plr_ate_coverage.csv b/results/plm/plr_ate_coverage.csv index c36f372..e1067cc 100644 --- a/results/plm/plr_ate_coverage.csv +++ b/results/plm/plr_ate_coverage.csv @@ -3,15 +3,15 @@ Lasso,Lasso,IV-type,0.9,0.881,0.1393979255576113,0.0352891099128789,1000 Lasso,Lasso,IV-type,0.95,0.945,0.16610287331091153,0.0352891099128789,1000 Lasso,Lasso,partialling out,0.9,0.908,0.14646362984437974,0.034686755904342816,1000 Lasso,Lasso,partialling out,0.95,0.956,0.17452217926042807,0.034686755904342816,1000 -Lasso,Random Forest,IV-type,0.9,0.902,0.14667137241231934,0.03601868567146106,1000 -Lasso,Random Forest,IV-type,0.95,0.955,0.1747697197981062,0.03601868567146106,1000 -Lasso,Random Forest,partialling out,0.9,0.823,0.14326184144338033,0.04187221910505238,1000 -Lasso,Random Forest,partialling out,0.95,0.887,0.1707070130661524,0.04187221910505238,1000 -Random Forest,Lasso,IV-type,0.9,0.879,0.1419251268889888,0.035737103404494694,1000 -Random Forest,Lasso,IV-type,0.95,0.954,0.16911421943315694,0.035737103404494694,1000 -Random Forest,Lasso,partialling out,0.9,0.91,0.15192780855925678,0.035963136818860576,1000 -Random Forest,Lasso,partialling out,0.95,0.95,0.18103314978739116,0.035963136818860576,1000 -Random Forest,Random Forest,IV-type,0.9,0.899,0.14940427990853258,0.036758863042842316,1000 -Random Forest,Random Forest,IV-type,0.95,0.951,0.1780261799340667,0.036758863042842316,1000 -Random Forest,Random Forest,partialling out,0.9,0.884,0.14635016172058904,0.03722964492247778,1000 -Random Forest,Random Forest,partialling out,0.95,0.937,0.17438697365162548,0.03722964492247778,1000 +Lasso,Random Forest,IV-type,0.9,0.895,0.14672332096879584,0.03621404411832195,1000 +Lasso,Random Forest,IV-type,0.95,0.953,0.17483162032109167,0.03621404411832195,1000 +Lasso,Random Forest,partialling out,0.9,0.816,0.14332193251141256,0.04209132271298422,1000 +Lasso,Random Forest,partialling out,0.95,0.888,0.17077861599008798,0.04209132271298422,1000 +Random Forest,Lasso,IV-type,0.9,0.883,0.14193144590758777,0.03591650651278451,1000 +Random Forest,Lasso,IV-type,0.95,0.951,0.16912174900823193,0.03591650651278451,1000 +Random Forest,Lasso,partialling out,0.9,0.901,0.1519648979095884,0.03615745065828101,1000 +Random Forest,Lasso,partialling out,0.95,0.952,0.181077344474182,0.03615745065828101,1000 +Random Forest,Random Forest,IV-type,0.9,0.893,0.14942288181113228,0.03672076481604481,1000 +Random Forest,Random Forest,IV-type,0.95,0.948,0.17804834546815554,0.03672076481604481,1000 +Random Forest,Random Forest,partialling out,0.9,0.878,0.14635785065361873,0.037417599679567926,1000 +Random Forest,Random Forest,partialling out,0.95,0.946,0.17439613558042621,0.037417599679567926,1000 diff --git a/results/plm/plr_ate_coverage_metadata.csv b/results/plm/plr_ate_coverage_metadata.csv index 99e9587..8d5c7fd 100644 --- a/results/plm/plr_ate_coverage_metadata.csv +++ b/results/plm/plr_ate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,plr_ate_coverage.py,2025-01-08 13:51:58,6877.582471370697,3.12.8 +0.10.dev0,plr_ate_coverage.py,2025-01-08 14:10:27,6916.502653360367,3.12.8 diff --git a/results/plm/plr_ate_sensitivity.csv b/results/plm/plr_ate_sensitivity.csv index 9e81866..00934b4 100644 --- a/results/plm/plr_ate_sensitivity.csv +++ b/results/plm/plr_ate_sensitivity.csv @@ -3,15 +3,15 @@ LGBM,LGBM,IV-type,0.9,0.49,1.2836841849471865,0.642852034975727,1.0,0.998,0.0883 LGBM,LGBM,IV-type,0.95,0.65,1.529604050351386,0.642852034975727,1.0,1.0,0.0883267598773138,0.013790413915027113,1.3451446703513257,0.27137365708627864,500 LGBM,LGBM,partialling out,0.9,0.052,1.0555526522517718,0.9216896766797483,1.0,0.878,0.12290629723561408,0.0667859903596657,1.6463724503661545,0.2830593598175856,500 LGBM,LGBM,partialling out,0.95,0.114,1.2577685626857555,0.9216896766797483,1.0,0.962,0.12290629723561408,0.05210975377693211,1.6463724503661545,0.2830593598175856,500 -LGBM,Random Forest,IV-type,0.9,0.078,1.1838339830161526,0.9284629204120292,1.0,0.936,0.11706718159870112,0.058134359561696,1.6948353167191215,0.26598096146580197,500 -LGBM,Random Forest,IV-type,0.95,0.176,1.4106251962897096,0.9284629204120292,1.0,0.99,0.11706718159870112,0.04301383836086259,1.6948353167191215,0.26598096146580197,500 -LGBM,Random Forest,partialling out,0.9,0.086,1.2429894487532662,0.9950306719680438,1.0,0.928,0.11840599523128223,0.06014723707947923,1.8101975263360996,0.29100166237609104,500 -LGBM,Random Forest,partialling out,0.95,0.148,1.4811132813288146,0.9950306719680438,1.0,0.98,0.11840599523128223,0.045182462073214504,1.8101975263360996,0.29100166237609104,500 -Random Forest,LGBM,IV-type,0.9,0.566,1.899626850133317,0.8880539202684316,1.0,0.998,0.07200043735278143,0.018305683679739147,2.120147429487342,0.4656159493536836,500 -Random Forest,LGBM,IV-type,0.95,0.746,2.2635450044433725,0.8880539202684316,1.0,1.0,0.07200043735278143,0.008671199476776914,2.120147429487342,0.4656159493536836,500 -Random Forest,LGBM,partialling out,0.9,0.002,1.5140477405908301,1.5731546878983491,1.0,0.83,0.12821046992923293,0.08047445195178365,2.774729569778034,0.40407728189451836,500 -Random Forest,LGBM,partialling out,0.95,0.008,1.804099157401693,1.5731546878983491,1.0,0.948,0.12821046992923293,0.06696088607192313,2.774729569778034,0.40407728189451836,500 -Random Forest,Random Forest,IV-type,0.9,0.01,1.7146246887321768,1.6147802774438333,1.0,0.892,0.11921570180694094,0.07016680024004021,2.9441846644720684,0.3940869566060415,500 -Random Forest,Random Forest,IV-type,0.95,0.042,2.043101332455167,1.6147802774438333,1.0,0.972,0.11921570180694094,0.05632386808702946,2.9441846644720684,0.3940869566060415,500 -Random Forest,Random Forest,partialling out,0.9,0.0,1.7428116454944356,1.7404826195551244,1.0,0.828,0.12836069509728124,0.07843487138656559,3.0651903476419413,0.46471318404652623,500 -Random Forest,Random Forest,partialling out,0.95,0.02,2.0766881630293903,1.7404826195551244,1.0,0.93,0.12836069509728124,0.06432351155843387,3.0651903476419413,0.46471318404652623,500 +LGBM,Random Forest,IV-type,0.9,0.072,1.1846157067784349,0.9322070273917288,1.0,0.934,0.11754241223304673,0.058515658188839345,1.6980814469560257,0.26726499739118403,500 +LGBM,Random Forest,IV-type,0.95,0.16,1.4115566776050241,0.9322070273917288,1.0,0.994,0.11754241223304673,0.04334326180178758,1.6980814469560257,0.26726499739118403,500 +LGBM,Random Forest,partialling out,0.9,0.078,1.2415314353647366,0.9950962660593035,1.0,0.918,0.11842149116467734,0.06021024644530299,1.8099982510012884,0.29229694861359934,500 +LGBM,Random Forest,partialling out,0.95,0.15,1.479375951220122,0.9950962660593035,1.0,0.98,0.11842149116467734,0.04525082945770369,1.8099982510012884,0.29229694861359934,500 +Random Forest,LGBM,IV-type,0.9,0.554,1.900486623597139,0.8881666888956794,1.0,1.0,0.07201385567825232,0.01814220891090863,2.11967639571274,0.46266796209857214,500 +Random Forest,LGBM,IV-type,0.95,0.746,2.2645694877143114,0.8881666888956794,1.0,1.0,0.07201385567825232,0.008578785415219068,2.11967639571274,0.46266796209857214,500 +Random Forest,LGBM,partialling out,0.9,0.002,1.5126486531347854,1.573242725840773,1.0,0.818,0.12824967172299567,0.08055028818351519,2.774465314925881,0.4031005519125824,500 +Random Forest,LGBM,partialling out,0.95,0.008,1.8024320418722997,1.573242725840773,1.0,0.948,0.12824967172299567,0.06705003584560411,2.774465314925881,0.4031005519125824,500 +Random Forest,Random Forest,IV-type,0.9,0.012,1.7129890288584426,1.6194855098925158,1.0,0.892,0.11959162432568951,0.07060214596391444,2.948463222220921,0.39969580126755805,500 +Random Forest,Random Forest,IV-type,0.95,0.038,2.041152323503278,1.6194855098925158,1.0,0.972,0.11959162432568951,0.05677408319758889,2.948463222220921,0.39969580126755805,500 +Random Forest,Random Forest,partialling out,0.9,0.0,1.7381117353695532,1.7366675645551972,1.0,0.824,0.1281371703134689,0.07833372270549209,3.0610973455175285,0.4635968018257515,500 +Random Forest,Random Forest,partialling out,0.95,0.018,2.0710878746970973,1.7366675645551972,1.0,0.946,0.1281371703134689,0.0642547167440049,3.0610973455175285,0.4635968018257515,500 diff --git a/results/plm/plr_ate_sensitivity_metadata.csv b/results/plm/plr_ate_sensitivity_metadata.csv index a8d0acc..f0d1e74 100644 --- a/results/plm/plr_ate_sensitivity_metadata.csv +++ b/results/plm/plr_ate_sensitivity_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,plr_ate_sensitivity.py,2025-01-08 16:11:17,15234.57352733612,3.12.8 +0.10.dev0,plr_ate_sensitivity.py,2025-01-08 16:36:12,15659.224347829819,3.12.8 diff --git a/results/plm/plr_cate_coverage_metadata.csv b/results/plm/plr_cate_coverage_metadata.csv index d4dbd53..59b490d 100644 --- a/results/plm/plr_cate_coverage_metadata.csv +++ b/results/plm/plr_cate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,plr_cate_coverage.py,2025-01-08 13:42:17,6295.241044282913,3.12.8 +0.10.dev0,plr_cate_coverage.py,2025-01-08 14:25:31,7808.868787527084,3.12.8 diff --git a/results/plm/plr_gate_coverage_metadata.csv b/results/plm/plr_gate_coverage_metadata.csv index 58d3f59..febe93f 100644 --- a/results/plm/plr_gate_coverage_metadata.csv +++ b/results/plm/plr_gate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,plr_gate_coverage.py,2025-01-08 12:43:49,2787.3405332565308,3.12.8 +0.10.dev0,plr_gate_coverage.py,2025-01-08 13:02:47,2835.5379569530487,3.12.8 diff --git a/results/rdd/rdd_fuzzy_coverage.csv b/results/rdd/rdd_fuzzy_coverage.csv new file mode 100644 index 0000000..053bed7 --- /dev/null +++ b/results/rdd/rdd_fuzzy_coverage.csv @@ -0,0 +1,99 @@ +Method,fs specification,Learner g,Learner m,level,Coverage,CI Length,Bias,repetition +rdflex,cutoff,Global linear,Global linear,0.9,0.9444444444444444,10.550968654406873,2.5392776744200893,216 +rdflex,cutoff,Global linear,Global linear,0.95,0.9768518518518519,12.572254592024322,2.5392776744200893,216 +rdflex,cutoff,Global linear,LGBM,0.9,0.9444444444444444,10.708024190570491,2.6073374467941046,216 +rdflex,cutoff,Global linear,LGBM,0.95,0.9814814814814815,12.759397806113123,2.6073374467941046,216 +rdflex,cutoff,Global linear,Linear,0.9,0.9398148148148148,10.693313577661321,2.588240057526377,216 +rdflex,cutoff,Global linear,Linear,0.95,0.9768518518518519,12.741869029679734,2.588240057526377,216 +rdflex,cutoff,Global linear,Stacked,0.9,0.9490740740740741,10.542048532266264,2.612637626054498,216 +rdflex,cutoff,Global linear,Stacked,0.95,0.9722222222222222,12.561625610912065,2.612637626054498,216 +rdflex,cutoff,LGBM,Global linear,0.9,0.9351851851851852,2.0279514032790127,0.46622804442663607,216 +rdflex,cutoff,LGBM,Global linear,0.95,0.9675925925925926,2.41645314069128,0.46622804442663607,216 +rdflex,cutoff,LGBM,LGBM,0.9,0.9305555555555556,2.042881297411709,0.5024654331303038,216 +rdflex,cutoff,LGBM,LGBM,0.95,0.9722222222222222,2.43424320681851,0.5024654331303038,216 +rdflex,cutoff,LGBM,Linear,0.9,0.9537037037037037,2.080782411875394,0.4852661380399315,216 +rdflex,cutoff,LGBM,Linear,0.95,0.9907407407407407,2.4794051702331084,0.4852661380399315,216 +rdflex,cutoff,LGBM,Stacked,0.9,0.9074074074074074,2.0172921723310107,0.4728302559834236,216 +rdflex,cutoff,LGBM,Stacked,0.95,0.9675925925925926,2.403751883619732,0.4728302559834236,216 +rdflex,cutoff,Linear,Global linear,0.9,0.9444444444444444,10.55132710224951,2.5325716396404885,216 +rdflex,cutoff,Linear,Global linear,0.95,0.9768518518518519,12.572681708972839,2.5325716396404885,216 +rdflex,cutoff,Linear,LGBM,0.9,0.9444444444444444,10.753117458827687,2.620660533593667,216 +rdflex,cutoff,Linear,LGBM,0.95,0.9768518518518519,12.813129749357891,2.620660533593667,216 +rdflex,cutoff,Linear,Linear,0.9,0.9351851851851852,10.743662262390526,2.595884369295528,216 +rdflex,cutoff,Linear,Linear,0.95,0.9768518518518519,12.801863187896167,2.595884369295528,216 +rdflex,cutoff,Linear,Stacked,0.9,0.9444444444444444,10.401649719757378,2.4927224287383223,216 +rdflex,cutoff,Linear,Stacked,0.95,0.9768518518518519,12.394330107238822,2.4927224287383223,216 +rdflex,cutoff,Stacked,Global linear,0.9,0.9305555555555556,2.0983415675320316,0.4975883722319017,216 +rdflex,cutoff,Stacked,Global linear,0.95,0.9722222222222222,2.5003281947029072,0.4975883722319017,216 +rdflex,cutoff,Stacked,LGBM,0.9,0.9490740740740741,1.9927333689440805,0.46390758670023197,216 +rdflex,cutoff,Stacked,LGBM,0.95,0.9907407407407407,2.374488264442265,0.46390758670023197,216 +rdflex,cutoff,Stacked,Linear,0.9,0.9398148148148148,2.1047878672512663,0.4512766757906042,216 +rdflex,cutoff,Stacked,Linear,0.95,0.9814814814814815,2.5080094345872537,0.4512766757906042,216 +rdflex,cutoff,Stacked,Stacked,0.9,0.9398148148148148,2.018372202175261,0.4440441469354455,216 +rdflex,cutoff,Stacked,Stacked,0.95,0.9722222222222222,2.4050388185556275,0.4440441469354455,216 +rdflex,cutoff and score,Global linear,Global linear,0.9,0.9444444444444444,10.541731726173863,2.5466073798084476,216 +rdflex,cutoff and score,Global linear,Global linear,0.95,0.9768518518518519,12.561248113169402,2.5466073798084476,216 +rdflex,cutoff and score,Global linear,LGBM,0.9,0.9444444444444444,10.802363295011858,2.619847318234262,216 +rdflex,cutoff and score,Global linear,LGBM,0.95,0.9814814814814815,12.871809782479394,2.619847318234262,216 +rdflex,cutoff and score,Global linear,Linear,0.9,0.9444444444444444,10.716958765048908,2.583063844578886,216 +rdflex,cutoff and score,Global linear,Linear,0.95,0.9768518518518519,12.770044008248046,2.583063844578886,216 +rdflex,cutoff and score,Global linear,Stacked,0.9,0.9490740740740741,10.844485522189487,2.620657146170252,216 +rdflex,cutoff and score,Global linear,Stacked,0.95,0.9768518518518519,12.922001511922078,2.620657146170252,216 +rdflex,cutoff and score,LGBM,Global linear,0.9,0.9537037037037037,2.1504153242188013,0.4963799840481196,216 +rdflex,cutoff and score,LGBM,Global linear,0.95,0.9814814814814815,2.5623779029404297,0.4963799840481196,216 +rdflex,cutoff and score,LGBM,LGBM,0.9,0.9398148148148148,2.19952802355761,0.5288148720294119,216 +rdflex,cutoff and score,LGBM,LGBM,0.95,0.9675925925925926,2.62089929372583,0.5288148720294119,216 +rdflex,cutoff and score,LGBM,Linear,0.9,0.9444444444444444,2.194220945907011,0.47775420988075235,216 +rdflex,cutoff and score,LGBM,Linear,0.95,0.9768518518518519,2.6145755206630503,0.47775420988075235,216 +rdflex,cutoff and score,LGBM,Stacked,0.9,0.9629629629629629,2.1314900769464837,0.507224779882837,216 +rdflex,cutoff and score,LGBM,Stacked,0.95,0.9814814814814815,2.5398270799100526,0.507224779882837,216 +rdflex,cutoff and score,Linear,Global linear,0.9,0.9305555555555556,10.639058631470585,2.5797082020294453,216 +rdflex,cutoff and score,Linear,Global linear,0.95,0.9768518518518519,12.67722027384236,2.5797082020294453,216 +rdflex,cutoff and score,Linear,LGBM,0.9,0.9537037037037037,10.843507930073951,2.6816901211660595,216 +rdflex,cutoff and score,Linear,LGBM,0.95,0.9861111111111112,12.920836639068572,2.6816901211660595,216 +rdflex,cutoff and score,Linear,Linear,0.9,0.9259259259259259,10.720333460959493,2.5692901598060844,216 +rdflex,cutoff and score,Linear,Linear,0.95,0.9768518518518519,12.774065206447778,2.5692901598060844,216 +rdflex,cutoff and score,Linear,Stacked,0.9,0.9351851851851852,10.771248925216401,2.628942922146688,216 +rdflex,cutoff and score,Linear,Stacked,0.95,0.9722222222222222,12.834734724126767,2.628942922146688,216 +rdflex,cutoff and score,Stacked,Global linear,0.9,0.9259259259259259,2.1758583329529992,0.4954139177181787,216 +rdflex,cutoff and score,Stacked,Global linear,0.95,0.9722222222222222,2.5926951177735758,0.4954139177181787,216 +rdflex,cutoff and score,Stacked,LGBM,0.9,0.9490740740740741,2.14896811377524,0.49554947239669855,216 +rdflex,cutoff and score,Stacked,LGBM,0.95,0.9675925925925926,2.5606534453346264,0.49554947239669855,216 +rdflex,cutoff and score,Stacked,Linear,0.9,0.9444444444444444,2.2103831812726713,0.5009430339517977,216 +rdflex,cutoff and score,Stacked,Linear,0.95,0.9907407407407407,2.6338340119399084,0.5009430339517977,216 +rdflex,cutoff and score,Stacked,Stacked,0.9,0.9583333333333334,2.248723364922043,0.48994200235087687,216 +rdflex,cutoff and score,Stacked,Stacked,0.95,0.9814814814814815,2.6795191585585116,0.48994200235087687,216 +rdflex,interacted cutoff and score,Global linear,Global linear,0.9,0.9444444444444444,10.528753908366333,2.518597595494908,216 +rdflex,interacted cutoff and score,Global linear,Global linear,0.95,0.9768518518518519,12.545784089450876,2.518597595494908,216 +rdflex,interacted cutoff and score,Global linear,LGBM,0.9,0.9490740740740741,10.883820485971622,2.643607944035682,216 +rdflex,interacted cutoff and score,Global linear,LGBM,0.95,0.9814814814814815,12.968872012179936,2.643607944035682,216 +rdflex,interacted cutoff and score,Global linear,Linear,0.9,0.9490740740740741,10.679264733081846,2.5668725486471975,216 +rdflex,interacted cutoff and score,Global linear,Linear,0.95,0.9768518518518519,12.72512879884764,2.5668725486471975,216 +rdflex,interacted cutoff and score,Global linear,Stacked,0.9,0.9444444444444444,10.556663173569188,2.583268501046405,216 +rdflex,interacted cutoff and score,Global linear,Stacked,0.95,0.9722222222222222,12.57904003012321,2.583268501046405,216 +rdflex,interacted cutoff and score,LGBM,Global linear,0.9,0.9398148148148148,2.1382530370306743,0.4936600928743498,216 +rdflex,interacted cutoff and score,LGBM,Global linear,0.95,0.9814814814814815,2.5478856438921023,0.4936600928743498,216 +rdflex,interacted cutoff and score,LGBM,LGBM,0.9,0.9444444444444444,2.243092570442924,0.5335638194586231,216 +rdflex,interacted cutoff and score,LGBM,LGBM,0.95,0.9814814814814815,2.6728096531030787,0.5335638194586231,216 +rdflex,interacted cutoff and score,LGBM,Linear,0.9,0.9305555555555556,2.2051763510151194,0.5213516989837321,216 +rdflex,interacted cutoff and score,LGBM,Linear,0.95,0.9861111111111112,2.6276296910135963,0.5213516989837321,216 +rdflex,interacted cutoff and score,LGBM,Stacked,0.9,0.9444444444444444,2.126751117895573,0.5233380765223618,216 +rdflex,interacted cutoff and score,LGBM,Stacked,0.95,0.9861111111111112,2.5341802619125082,0.5233380765223618,216 +rdflex,interacted cutoff and score,Linear,Global linear,0.9,0.9351851851851852,10.677559557738437,2.5991484671227343,216 +rdflex,interacted cutoff and score,Linear,Global linear,0.95,0.9768518518518519,12.723096957104614,2.5991484671227343,216 +rdflex,interacted cutoff and score,Linear,LGBM,0.9,0.9490740740740741,11.089339149119295,2.725598433398943,216 +rdflex,interacted cutoff and score,Linear,LGBM,0.95,0.9814814814814815,13.213762603853324,2.725598433398943,216 +rdflex,interacted cutoff and score,Linear,Linear,0.9,0.9490740740740741,10.802881515458683,2.6051325493976867,216 +rdflex,interacted cutoff and score,Linear,Linear,0.95,0.9768518518518519,12.872427280228244,2.6051325493976867,216 +rdflex,interacted cutoff and score,Linear,Stacked,0.9,0.9305555555555556,10.739902046135317,2.632007340388683,216 +rdflex,interacted cutoff and score,Linear,Stacked,0.95,0.9722222222222222,12.797382613872113,2.632007340388683,216 +rdflex,interacted cutoff and score,Stacked,Global linear,0.9,0.9259259259259259,2.2262353572464413,0.5277443836534749,216 +rdflex,interacted cutoff and score,Stacked,Global linear,0.95,0.9722222222222222,2.6527230446636065,0.5277443836534749,216 +rdflex,interacted cutoff and score,Stacked,LGBM,0.9,0.9351851851851852,2.208657937129488,0.5111335387106734,216 +rdflex,interacted cutoff and score,Stacked,LGBM,0.95,0.9768518518518519,2.6317782567470007,0.5111335387106734,216 +rdflex,interacted cutoff and score,Stacked,Linear,0.9,0.9212962962962963,2.236992575930734,0.5108504373114083,216 +rdflex,interacted cutoff and score,Stacked,Linear,0.95,0.9675925925925926,2.6655410613245243,0.5108504373114083,216 +rdflex,interacted cutoff and score,Stacked,Stacked,0.9,0.9490740740740741,2.1809709471862324,0.48273354169573274,216 +rdflex,interacted cutoff and score,Stacked,Stacked,0.95,0.9861111111111112,2.598787173382533,0.48273354169573274,216 +rdrobust,cutoff,linear,linear,0.9,0.9398148148148148,10.388284844054892,2.528176524108971,216 +rdrobust,cutoff,linear,linear,0.95,0.9722222222222222,12.378404875591745,2.528176524108971,216 diff --git a/results/rdd/rdd_fuzzy_coverage_metadata.csv b/results/rdd/rdd_fuzzy_coverage_metadata.csv new file mode 100644 index 0000000..d3b74a5 --- /dev/null +++ b/results/rdd/rdd_fuzzy_coverage_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (seconds),Python Version +0.9.dev0,rdd_fuzzy_coverage.py,2025-01-08 23:31:57,19859.63268303871,3.12.8 diff --git a/results/rdd/rdd_sharp_coverage.csv b/results/rdd/rdd_sharp_coverage.csv new file mode 100644 index 0000000..558f06e --- /dev/null +++ b/results/rdd/rdd_sharp_coverage.csv @@ -0,0 +1,27 @@ +Method,fs specification,Learner g,level,Coverage,CI Length,Bias,repetition +rdflex,cutoff,Global linear,0.9,0.874,2.2011088216647567,0.5534513823633507,500 +rdflex,cutoff,Global linear,0.95,0.922,2.6227829308507813,0.5534513823633507,500 +rdflex,cutoff,LGBM,0.9,0.914,0.5720094363002372,0.1386055523323647,500 +rdflex,cutoff,LGBM,0.95,0.96,0.681591283014874,0.1386055523323647,500 +rdflex,cutoff,Linear,0.9,0.874,2.2135060296971463,0.558126057195101,500 +rdflex,cutoff,Linear,0.95,0.918,2.6375551153504855,0.558126057195101,500 +rdflex,cutoff,Stacked,0.9,0.9,0.5590842827226126,0.12977180379581715,500 +rdflex,cutoff,Stacked,0.95,0.964,0.6661900125968221,0.12977180379581715,500 +rdflex,cutoff and score,Global linear,0.9,0.876,2.2005650015589864,0.5551812535814453,500 +rdflex,cutoff and score,Global linear,0.95,0.922,2.622134929226859,0.5551812535814453,500 +rdflex,cutoff and score,LGBM,0.9,0.902,0.5984294788353389,0.14476299319184177,500 +rdflex,cutoff and score,LGBM,0.95,0.95,0.7130727054286046,0.14476299319184177,500 +rdflex,cutoff and score,Linear,0.9,0.87,2.212887444081595,0.5582781571131341,500 +rdflex,cutoff and score,Linear,0.95,0.922,2.63681802512679,0.5582781571131341,500 +rdflex,cutoff and score,Stacked,0.9,0.88,0.5820101876494705,0.14231610717047102,500 +rdflex,cutoff and score,Stacked,0.95,0.956,0.6935079132497274,0.14231610717047102,500 +rdflex,interacted cutoff and score,Global linear,0.9,0.878,2.202268208292408,0.5546663773642981,500 +rdflex,interacted cutoff and score,Global linear,0.95,0.926,2.6241644252264025,0.5546663773642981,500 +rdflex,interacted cutoff and score,LGBM,0.9,0.886,0.6002731299225453,0.151487055237104,500 +rdflex,interacted cutoff and score,LGBM,0.95,0.948,0.7152695511975984,0.151487055237104,500 +rdflex,interacted cutoff and score,Linear,0.9,0.88,2.2252512728137637,0.5541600195953326,500 +rdflex,interacted cutoff and score,Linear,0.95,0.916,2.651550435737079,0.5541600195953326,500 +rdflex,interacted cutoff and score,Stacked,0.9,0.904,0.5793073094025809,0.14171607962579613,500 +rdflex,interacted cutoff and score,Stacked,0.95,0.962,0.6902872351713266,0.14171607962579613,500 +rdrobust,cutoff,linear,0.9,0.874,2.1797037623552287,0.555486091306879,500 +rdrobust,cutoff,linear,0.95,0.916,2.597277229525019,0.555486091306879,500 diff --git a/results/rdd/rdd_sharp_coverage_metadata.csv b/results/rdd/rdd_sharp_coverage_metadata.csv new file mode 100644 index 0000000..8d0e821 --- /dev/null +++ b/results/rdd/rdd_sharp_coverage_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (seconds),Python Version +0.9.dev0,rdd_sharp_coverage.py,2025-01-08 19:08:43,4069.1944484710693,3.12.8 diff --git a/scripts/rdd/rdd_fuzzy_coverage.py b/scripts/rdd/rdd_fuzzy_coverage.py new file mode 100644 index 0000000..f9773fa --- /dev/null +++ b/scripts/rdd/rdd_fuzzy_coverage.py @@ -0,0 +1,177 @@ +import numpy as np +import pandas as pd +from datetime import datetime +import time +import sys + +from lightgbm import LGBMRegressor, LGBMClassifier +from sklearn.ensemble import StackingRegressor, StackingClassifier +from sklearn.linear_model import LinearRegression, Ridge, LogisticRegression +from rdrobust import rdrobust + +import doubleml as dml +from doubleml.rdd import RDFlex +from doubleml.rdd.datasets import make_simple_rdd_data +from doubleml.utils import GlobalRegressor, GlobalClassifier + +from statsmodels.nonparametric.kernel_regression import KernelReg + + +# Number of repetitions +n_rep = 500 +max_runtime = 5.5 * 3600 # 5.5 hours in seconds + +# DGP pars +n_obs = 2000 +cutoff = 0 + +# to get the best possible comparison between different learners (and settings) we first simulate all datasets +np.random.seed(42) + +datasets = [] +for i in range(n_rep): + data = make_simple_rdd_data(n_obs=n_obs, fuzzy=True, cutoff=cutoff) + datasets.append(data) + +# set up hyperparameters +hyperparam_dict = { + "fs_specification": ["cutoff", "cutoff and score", "interacted cutoff and score"], + "learner_g": [ + ("Linear", LinearRegression()), + ("LGBM", LGBMRegressor(n_estimators=100, max_depth=5, learning_rate=0.1, verbose=-1)), + ("Global linear", GlobalRegressor(LinearRegression())), + ("Stacked", StackingRegressor( + estimators=[ + ('lr', LinearRegression()), + ('lgbm', LGBMRegressor(n_estimators=100, max_depth=5, learning_rate=0.1, verbose=-1)), + ('glr', GlobalRegressor(LinearRegression()))], + final_estimator=Ridge()))], + "learner_m": [ + ("Linear", LogisticRegression()), + ("LGBM", LGBMClassifier(n_estimators=100, max_depth=5, learning_rate=0.1, verbose=-1)), + ("Global linear", GlobalClassifier(LogisticRegression())), + ("Stacked", StackingClassifier( + estimators=[ + ('lr', LogisticRegression()), + ('lgbm', LGBMClassifier(n_estimators=100, max_depth=5, learning_rate=0.1, verbose=-1)), + ('glr', GlobalClassifier(LogisticRegression()))], + final_estimator=LogisticRegression()))], + "level": [0.95, 0.90]} + +# set up the results dataframe +df_results_detailed = pd.DataFrame() + +# start simulation +np.random.seed(42) +start_time = time.time() + +for i_rep in range(n_rep): + print(f"Repetition: {i_rep}/{n_rep}", end="\r") + + # Check the elapsed time + elapsed_time = time.time() - start_time + if elapsed_time > max_runtime: + print("Maximum runtime exceeded. Stopping the simulation.") + break + + data = datasets[i_rep] + # get oracle value + score = data["score"] + complier_mask = (((data["D"] == 0) & (data["score"] < cutoff)) | ((data["D"] == 1) & (data["score"] > cutoff))) + + ite = data["oracle_values"]['Y1'] - data["oracle_values"]['Y0'] + kernel_reg = KernelReg(endog=ite[complier_mask], exog=score[complier_mask], var_type='c', reg_type='ll') + effect_at_cutoff, _ = kernel_reg.fit(np.array([cutoff])) + oracle_effect = effect_at_cutoff[0] + + Y = data["Y"] + Z = data["X"].reshape(n_obs, -1) + D = data["D"] + + # baseline + for level_idx, level in enumerate(hyperparam_dict["level"]): + res = rdrobust(y=Y, x=score, fuzzy=D, covs=Z, c=cutoff, level=level*100) + coef = res.coef.loc["Robust", "Coeff"] + ci_lower = res.ci.loc["Robust", "CI Lower"] + ci_upper = res.ci.loc["Robust", "CI Upper"] + + coverage = (ci_lower < oracle_effect) & (oracle_effect < ci_upper) + ci_length = ci_upper - ci_lower + + df_results_detailed = pd.concat( + (df_results_detailed, + pd.DataFrame({ + "Coverage": coverage.astype(int), + "CI Length": ci_length, + "Bias": abs(coef - oracle_effect), + "Learner g": "linear", + "Learner m": "linear", + "Method": "rdrobust", + "fs specification": "cutoff", + "level": level, + "repetition": i_rep}, index=[0])), + ignore_index=True) + + # define the DoubleML data object + obj_dml_data = dml.DoubleMLData.from_arrays(y=Y, d=D, x=Z, s=score) + + for learner_g_idx, (learner_g_name, ml_g) in enumerate(hyperparam_dict["learner_g"]): + for learner_m_idx, (learner_m_name, ml_m) in enumerate(hyperparam_dict["learner_m"]): + for fs_specification_idx, fs_specification in enumerate(hyperparam_dict["fs_specification"]): + rdflex_model = RDFlex( + obj_dml_data, + ml_g=ml_g, + ml_m=ml_m, + n_folds=5, + n_rep=1, + cutoff=cutoff, + fuzzy=True, + fs_specification=fs_specification) + rdflex_model.fit(n_iterations=2) + + for level_idx, level in enumerate(hyperparam_dict["level"]): + confint = rdflex_model.confint(level=level) + coverage = (confint.iloc[2, 0] < oracle_effect) & (oracle_effect < confint.iloc[2, 1]) + ci_length = confint.iloc[2, 1] - confint.iloc[2, 0] + + df_results_detailed = pd.concat( + (df_results_detailed, + pd.DataFrame({ + "Coverage": coverage.astype(int), + "CI Length": ci_length, + "Bias": abs(rdflex_model.coef[2] - oracle_effect), + "Learner g": learner_g_name, + "Learner m": learner_m_name, + "Method": "rdflex", + "fs specification": fs_specification, + "level": level, + "repetition": i_rep}, index=[0])), + ignore_index=True) + +df_results = df_results_detailed.groupby( + ["Method", "fs specification", "Learner g", "Learner m", "level"]).agg( + {"Coverage": "mean", + "CI Length": "mean", + "Bias": "mean", + "repetition": "count"} + ).reset_index() +print(df_results) + +end_time = time.time() +total_runtime = end_time - start_time + +# save results +script_name = "rdd_fuzzy_coverage.py" +path = "results/rdd/rdd_fuzzy_coverage" + +metadata = pd.DataFrame({ + 'DoubleML Version': [dml.__version__], + 'Script': [script_name], + 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], + 'Total Runtime (seconds)': [total_runtime], + 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], +}) +print(metadata) + +df_results.to_csv(f"{path}.csv", index=False) +metadata.to_csv(f"{path}_metadata.csv", index=False) diff --git a/scripts/rdd/rdd_sharp_coverage.py b/scripts/rdd/rdd_sharp_coverage.py new file mode 100644 index 0000000..bff96f9 --- /dev/null +++ b/scripts/rdd/rdd_sharp_coverage.py @@ -0,0 +1,162 @@ +import numpy as np +import pandas as pd +from datetime import datetime +import time +import sys + +from lightgbm import LGBMRegressor +from sklearn.ensemble import StackingRegressor +from sklearn.linear_model import LinearRegression, Ridge +from rdrobust import rdrobust + +import doubleml as dml +from doubleml.rdd import RDFlex +from doubleml.rdd.datasets import make_simple_rdd_data +from doubleml.utils import GlobalRegressor + +from statsmodels.nonparametric.kernel_regression import KernelReg + + +# Number of repetitions +n_rep = 500 +max_runtime = 5.5 * 3600 # 5.5 hours in seconds + +# DGP pars +n_obs = 1000 +cutoff = 0 + +# to get the best possible comparison between different learners (and settings) we first simulate all datasets +np.random.seed(42) + +datasets = [] +for i in range(n_rep): + data = make_simple_rdd_data(n_obs=n_obs, fuzzy=False, cutoff=cutoff) + datasets.append(data) + +# set up hyperparameters +hyperparam_dict = { + "fs_specification": ["cutoff", "cutoff and score", "interacted cutoff and score"], + "learner_g": [ + ("Linear", LinearRegression()), + ("LGBM", LGBMRegressor(n_estimators=100, max_depth=5, learning_rate=0.1, verbose=-1)), + ("Global linear", GlobalRegressor(LinearRegression())), + ("Stacked", StackingRegressor( + estimators=[ + ('lr', LinearRegression()), + ('lgbm', LGBMRegressor(n_estimators=100, max_depth=5, learning_rate=0.1, verbose=-1)), + ('glr', GlobalRegressor(LinearRegression()))], + final_estimator=Ridge()))], + "level": [0.95, 0.90]} + +# set up the results dataframe +df_results_detailed = pd.DataFrame() + +# start simulation +np.random.seed(42) +start_time = time.time() + +for i_rep in range(n_rep): + print(f"Repetition: {i_rep}/{n_rep}", end="\r") + + # Check the elapsed time + elapsed_time = time.time() - start_time + if elapsed_time > max_runtime: + print("Maximum runtime exceeded. Stopping the simulation.") + break + + data = datasets[i_rep] + # get oracle value + score = data["score"] + ite = data["oracle_values"]['Y1'] - data["oracle_values"]['Y0'] + + kernel_reg = KernelReg(endog=ite, exog=score, var_type='c', reg_type='ll') + effect_at_cutoff, _ = kernel_reg.fit(np.array([cutoff])) + oracle_effect = effect_at_cutoff[0] + + Y = data["Y"] + Z = data["X"].reshape(n_obs, -1) + D = data["D"] + + # baseline + for level_idx, level in enumerate(hyperparam_dict["level"]): + res = rdrobust(y=Y, x=score, covs=Z, c=cutoff, level=level*100) + coef = res.coef.loc["Robust", "Coeff"] + ci_lower = res.ci.loc["Robust", "CI Lower"] + ci_upper = res.ci.loc["Robust", "CI Upper"] + + coverage = (ci_lower < oracle_effect) & (oracle_effect < ci_upper) + ci_length = ci_upper - ci_lower + + df_results_detailed = pd.concat( + (df_results_detailed, + pd.DataFrame({ + "Coverage": coverage.astype(int), + "CI Length": ci_length, + "Bias": abs(coef - oracle_effect), + "Learner g": "linear", + "Method": "rdrobust", + "fs specification": "cutoff", + "level": level, + "repetition": i_rep}, index=[0])), + ignore_index=True) + + # define the DoubleML data object + obj_dml_data = dml.DoubleMLData.from_arrays(y=Y, d=D, x=Z, s=score) + + for learner_g_idx, (learner_g_name, ml_g) in enumerate(hyperparam_dict["learner_g"]): + for fs_specification_idx, fs_specification in enumerate(hyperparam_dict["fs_specification"]): + rdflex_model = RDFlex( + obj_dml_data, + ml_g=ml_g, + n_folds=5, + n_rep=1, + cutoff=cutoff, + fuzzy=False, + fs_specification=fs_specification) + rdflex_model.fit(n_iterations=2) + + for level_idx, level in enumerate(hyperparam_dict["level"]): + confint = rdflex_model.confint(level=level) + coverage = (confint.iloc[2, 0] < oracle_effect) & (oracle_effect < confint.iloc[2, 1]) + ci_length = confint.iloc[2, 1] - confint.iloc[2, 0] + + df_results_detailed = pd.concat( + (df_results_detailed, + pd.DataFrame({ + "Coverage": coverage.astype(int), + "CI Length": ci_length, + "Bias": abs(rdflex_model.coef[2] - oracle_effect), + "Learner g": learner_g_name, + "Method": "rdflex", + "fs specification": fs_specification, + "level": level, + "repetition": i_rep}, index=[0])), + ignore_index=True) + +df_results = df_results_detailed.groupby( + ["Method", "fs specification", "Learner g", "level"]).agg( + {"Coverage": "mean", + "CI Length": "mean", + "Bias": "mean", + "repetition": "count"} + ).reset_index() +print(df_results) + +end_time = time.time() +total_runtime = end_time - start_time + +# save results +script_name = "rdd_sharp_coverage.py" +path = "results/rdd/rdd_sharp_coverage" + +metadata = pd.DataFrame({ + 'DoubleML Version': [dml.__version__], + 'Script': [script_name], + 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], + 'Total Runtime (seconds)': [total_runtime], + 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], +}) +print(metadata) + +df_results.to_csv(f"{path}.csv", index=False) +metadata.to_csv(f"{path}_metadata.csv", index=False)