Using RandomizedCV with xgboost

[dma092]

Hi all,

I am trying to use randomized cross validation on dask-xgboost.
Here is the snippet of code I am trying to get to work:

from dask_ml.model_selection import RandomizedSearchCV
from dask_ml.datasets import make_classification
from scipy.stats import uniform

X, y = make_classification(chunks=50)

model=dxgboost.XGBClassifier()
client=Client()
metric="f1"
data=X
target=y
bounds={"base_score": uniform(0.3, 0.7), "max_depth": uniform(3, 40), "learning_rate": uniform(0.05, 0.4),
                       "n_estimators": uniform(50, 200), "gamma": uniform(0, 10)}
results=RandomizedSearchCV(model, bounds,cv=3,scoring=metric,scheduler=client.scheduler).fit(data,target).best_estimator_

While running this code, I get the error TypeError: 'Future' object does not support indexing.
I think that I am misusing the api. If somebody can confirm that and point me to the right direction, I will be gratefull.
Thanks

[TomAugspurger]

FYI, I think you want scheduler=client, not scheduler=client.scheduler.

[TomAugspurger]

That works correctly for RandomizedSearchCV[SGDClasfifier]. For RandomizedSearchCV[SGBClassfier] That gives a different error

ValueError                                Traceback (most recent call last)
<ipython-input-10-6b15c84df156> in <module>()
      9                            scoring=metric,
     10                            scheduler=client,
---> 11                            ).fit(data,target).best_estimator_

~/miniconda3/envs/dask-xgboost/lib/python3.6/site-packages/dask_searchcv/model_selection.py in fit(self, X, y, groups, **fit_params)
    865         scheduler = _normalize_scheduler(self.scheduler, n_jobs)
    866 
--> 867         out = scheduler(dsk, keys, num_workers=n_jobs)
    868 
    869         results = handle_deprecated_train_score(out[0], self.return_train_score)

~/miniconda3/envs/dask-xgboost/lib/python3.6/site-packages/distributed/client.py in get(self, dsk, keys, restrictions, loose_restrictions, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, **kwargs)
   2157             try:
   2158                 results = self.gather(packed, asynchronous=asynchronous,
-> 2159                                       direct=direct)
   2160             finally:
   2161                 for f in futures.values():

~/miniconda3/envs/dask-xgboost/lib/python3.6/site-packages/distributed/client.py in gather(self, futures, errors, maxsize, direct, asynchronous)
   1560             return self.sync(self._gather, futures, errors=errors,
   1561                              direct=direct, local_worker=local_worker,
-> 1562                              asynchronous=asynchronous)
   1563 
   1564     @gen.coroutine

~/miniconda3/envs/dask-xgboost/lib/python3.6/site-packages/distributed/client.py in sync(self, func, *args, **kwargs)
    650             return future
    651         else:
--> 652             return sync(self.loop, func, *args, **kwargs)
    653 
    654     def __repr__(self):

~/miniconda3/envs/dask-xgboost/lib/python3.6/site-packages/distributed/utils.py in sync(loop, func, *args, **kwargs)
    273             e.wait(10)
    274     if error[0]:
--> 275         six.reraise(*error[0])
    276     else:
    277         return result[0]

~/miniconda3/envs/dask-xgboost/lib/python3.6/site-packages/six.py in reraise(tp, value, tb)
    691             if value.__traceback__ is not tb:
    692                 raise value.with_traceback(tb)
--> 693             raise value
    694         finally:
    695             value = None

~/miniconda3/envs/dask-xgboost/lib/python3.6/site-packages/distributed/utils.py in f()
    258             yield gen.moment
    259             thread_state.asynchronous = True
--> 260             result[0] = yield make_coro()
    261         except Exception as exc:
    262             error[0] = sys.exc_info()

~/miniconda3/envs/dask-xgboost/lib/python3.6/site-packages/tornado/gen.py in run(self)
   1097 
   1098                     try:
-> 1099                         value = future.result()
   1100                     except Exception:
   1101                         self.had_exception = True

~/miniconda3/envs/dask-xgboost/lib/python3.6/site-packages/tornado/gen.py in run(self)
   1105                     if exc_info is not None:
   1106                         try:
-> 1107                             yielded = self.gen.throw(*exc_info)
   1108                         finally:
   1109                             # Break up a reference to itself

~/miniconda3/envs/dask-xgboost/lib/python3.6/site-packages/distributed/client.py in _gather(self, futures, errors, direct, local_worker)
   1437                             six.reraise(type(exception),
   1438                                         exception,
-> 1439                                         traceback)
   1440                     if errors == 'skip':
   1441                         bad_keys.add(key)

~/miniconda3/envs/dask-xgboost/lib/python3.6/site-packages/six.py in reraise(tp, value, tb)
    690                 value = tp()
    691             if value.__traceback__ is not tb:
--> 692                 raise value.with_traceback(tb)
    693             raise value
    694         finally:

~/miniconda3/envs/dask-xgboost/lib/python3.6/site-packages/dask_searchcv/methods.py in fit_and_score()
    278     y_test = cv.extract(X, y, n, False, False)
    279     est_and_time = fit(est, X_train, y_train, error_score,
--> 280                        fields, params, fit_params)
    281     if not return_train_score:
    282         X_train = y_train = None

~/miniconda3/envs/dask-xgboost/lib/python3.6/site-packages/dask_searchcv/methods.py in fit()
    214         try:
    215             est = set_params(est, fields, params)
--> 216             est.fit(X, y, **fit_params)
    217         except Exception as e:
    218             if error_score == 'raise':

~/miniconda3/envs/dask-xgboost/lib/python3.6/site-packages/dask_xgboost/core.py in fit()
    273         3. The ``classes_`` and ``n_classes_`` attributes are not learned
    274         """
--> 275         client = default_client()
    276         xgb_options = self.get_xgb_params()
    277         self._Booster = train(client, xgb_options, X, y,

~/miniconda3/envs/dask-xgboost/lib/python3.6/site-packages/distributed/client.py in default_client()
   3609         return c
   3610     else:
-> 3611         raise ValueError("No clients found\n"
   3612                          "Start an client and point it to the scheduler address\n"
   3613                          "  from distributed import Client\n"

ValueError: No clients found
Start an client and point it to the scheduler address
  from distributed import Client
  client = Client('ip-addr-of-scheduler:8786')

[TomAugspurger]

There may be two issues.

"No clients found", and a secondary issue once that's resolved:

It seems like by the time it gets to dask_searchcv.methods.fit_and_score the values are NumPy arrays, while dask-xgboost only expects dask arrays (or dataframes).

I'm debugging with

from dask_ml.model_selection import RandomizedSearchCV
from dask_ml.datasets import make_classification
from scipy.stats import uniform
import dask_ml.xgboost as dxgboost
from distributed import Client


def main():
    client=Client(processes=False)
    X, y = make_classification(chunks=50)

    model = dxgboost.XGBClassifier()
    metric="f1"
    data=X
    target=y
    bounds={"max_depth": uniform(0.3, 0.7)}
    results=RandomizedSearchCV(model,
                               bounds,
                               cv=3,
                               scoring=metric,
                               scheduler=client,
                               ).fit(data,target).best_estimator_


__spec__ = 'bug.py'

if __name__ == '__main__':
    main()

I likely won't have time to debug this further right now. Would welcome additional help here if you have time @dma092

[mrocklin]

I would just leave scheduler= blank and let the default behavior handle things

…

On Thu, Jun 28, 2018 at 10:49 AM, Tom Augspurger ***@***.***> wrote: There may be two issues. "No clients found", and a secondary issue once that's resolved: It seems like by the time it gets to dask_searchcv.methods.fit_and_score the values are NumPy arrays, while dask-xgboost only expects dask arrays (or dataframes). I'm debugging with from dask_ml.model_selection import RandomizedSearchCVfrom dask_ml.datasets import make_classificationfrom scipy.stats import uniformimport dask_ml.xgboost as dxgboostfrom distributed import Client def main(): client=Client(processes=False) X, y = make_classification(chunks=50) model = dxgboost.XGBClassifier() metric="f1" data=X target=y bounds={"max_depth": uniform(0.3, 0.7)} results=RandomizedSearchCV(model, bounds, cv=3, scoring=metric, scheduler=client, ).fit(data,target).best_estimator_ __spec__ = 'bug.py' if __name__ == '__main__': main() I likely won't have time to debug this further right now. Would welcome additional help here if you have time @dma092 <https://github.com/dma092> — You are receiving this because you are subscribed to this thread. Reply to this email directly, view it on GitHub <#21 (comment)>, or mute the thread <https://github.com/notifications/unsubscribe-auth/AASszK8SdsZ457KYOrxs0ZGJXh0f3QZbks5uBOz5gaJpZM4U7etR> .

[mrocklin]

Long term it looks like dask-searchcv needs to update to the newer scheduler= keyword handling in dask On Thu, Jun 28, 2018 at 10:53 AM, Matthew Rocklin <mrocklin@anaconda.com> wrote:

…

I would just leave scheduler= blank and let the default behavior handle things On Thu, Jun 28, 2018 at 10:49 AM, Tom Augspurger ***@***.*** > wrote: > There may be two issues. > > "No clients found", and a secondary issue once that's resolved: > > It seems like by the time it gets to dask_searchcv.methods.fit_and_score > the values are NumPy arrays, while dask-xgboost only expects dask arrays > (or dataframes). > > I'm debugging with > > from dask_ml.model_selection import RandomizedSearchCVfrom dask_ml.datasets import make_classificationfrom scipy.stats import uniformimport dask_ml.xgboost as dxgboostfrom distributed import Client > > def main(): > client=Client(processes=False) > X, y = make_classification(chunks=50) > > model = dxgboost.XGBClassifier() > metric="f1" > data=X > target=y > bounds={"max_depth": uniform(0.3, 0.7)} > results=RandomizedSearchCV(model, > bounds, > cv=3, > scoring=metric, > scheduler=client, > ).fit(data,target).best_estimator_ > > __spec__ = 'bug.py' > if __name__ == '__main__': > main() > > I likely won't have time to debug this further right now. Would welcome > additional help here if you have time @dma092 <https://github.com/dma092> > > — > You are receiving this because you are subscribed to this thread. > Reply to this email directly, view it on GitHub > <#21 (comment)>, > or mute the thread > <https://github.com/notifications/unsubscribe-auth/AASszK8SdsZ457KYOrxs0ZGJXh0f3QZbks5uBOz5gaJpZM4U7etR> > . >

[dma092]

@TomAugspurger
I would be glad to!

[furlongce]

Any idea when this will be resolved?

[jcuda000001]

@TomAugspurger Is there currently any other approaches to run Hyperparameter tuning for XGBoost in Dask-ML where the workload is primarily "memory constrained"? It seems both GridSearchCV and RandomizedCV with XGBoost in Dask-ML ends up in the "No clients found" error. Does XGBoost support partial_fit API to run in IncrementalSearchCV? Thanks for all the work related to Dask-ML.

[TomAugspurger]

I don't think XGBoost implements partial_fit, but it's possible to add on your own by subclassing their estimators and using `xgb_model=self._booster` on subsequent calls. I don't have an example handy at the moment unfortunately.

…

On Mon, Jun 8, 2020 at 4:15 PM jcuda000001 ***@***.***> wrote: @TomAugspurger <https://github.com/TomAugspurger> Is there currently any other approaches to run Hyperparameter tuning for XGBoost in Dask-ML where the workload is primarily "memory constrained"? It seems both GridSearchCV and RandomizedCV with XGBoost in Dask-ML ends up in the "No clients found" error. Does XGBoost support partial_fit API to run in IncrementalSearchCV? Thanks for all the work related to Dask-ML. — You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub <#21 (comment)>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AAKAOIVNCODPJGUI427TUBDRVVIIBANCNFSM4FHN5NIQ> .

[jcuda000001]

@TomAugspurger - Thanks. Just to confirm, is there any hyperparameter tuning modules in Dask/Dask-ML which works currently for algorithms without partial_fit already implemented?

[TomAugspurger]

Yes, GridSearchCV and RandomizedSearchCV both work with estimators that implement `.fit`. They just don't work with the latest version of xgboost's estimators because of that bug in the type of exception raised, which is fixed on xgboost master.

…

On Tue, Jun 9, 2020 at 7:17 AM jcuda000001 ***@***.***> wrote: @TomAugspurger <https://github.com/TomAugspurger> - Thanks. Just to confirm, is there any hyperparameter tuning modules in Dask/Dask-ML which works currently for algorithms without partial_fit already implemented? — You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub <#21 (comment)>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AAKAOIT3RMAKLIYBD7EOCFTRVYR4TANCNFSM4FHN5NIQ> .